crypto_quant/huge_volume_main.py

from core.huge_volume import HugeVolume
from core.db_market_data import DBMarketData
from core.db_huge_volume_data import DBHugeVolumeData
from core.utils import timestamp_to_datetime
from market_data_main import MarketDataMain
import logging
from config import MONITOR_CONFIG, MYSQL_CONFIG, WINDOW_SIZE
from datetime import datetime
import pandas as pd
import os
import re

logging.basicConfig(
    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)


class HugeVolumeMain:
    def __init__(self, threshold: float = 2.0):
        mysql_user = MYSQL_CONFIG.get("user", "xch")
        mysql_password = MYSQL_CONFIG.get("password", "")
        if not mysql_password:
            raise ValueError("MySQL password is not set")
        mysql_host = MYSQL_CONFIG.get("host", "localhost")
        mysql_port = MYSQL_CONFIG.get("port", 3306)
        mysql_database = MYSQL_CONFIG.get("database", "okx")

        self.db_url = f"mysql+pymysql://{mysql_user}:{mysql_password}@{mysql_host}:{mysql_port}/{mysql_database}"
        self.huge_volume = HugeVolume()
        self.db_market_data = DBMarketData(self.db_url)
        self.db_huge_volume_data = DBHugeVolumeData(self.db_url)
        self.market_data_main = MarketDataMain()
        self.threshold = threshold

        self.output_folder = "./output/huge_volume_statistics/"
        os.makedirs(self.output_folder, exist_ok=True)

    def batch_initial_detect_volume_spike(self, window_size: int = 50, start: str = None):
        for symbol in self.market_data_main.symbols:
            for bar in self.market_data_main.bars:
                if start is None:
                    start = MONITOR_CONFIG.get("volume_monitor", {}).get(
                        "initial_date", "2025-05-01 00:00:00"
                    )
                data = self.detect_volume_spike(
                    symbol, bar, window_size, start, only_output_huge_volume=True, is_update=False
                )
                if data is not None and len(data) > 0:
                    logging.info(f"此次初始化巨量交易数据: {len(data)}条")
                else:
                    logging.info(f"此次初始化巨量交易数据为空")

    def detect_volume_spike(
        self,
        symbol: str = "XCH-USDT",
        bar: str = "5m",
        window_size: int = 50,
        start: str = "2025-05-01 00:00:00",
        end: str = None,
        only_output_huge_volume: bool = False,
        is_update: bool = False,
    ):
        if start is None:
            start = MONITOR_CONFIG.get("volume_monitor", {}).get(
                "initial_date", "2025-05-01 00:00:00"
            )
        if end is None:
            end = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        logging.info(f"开始处理巨量交易数据: {symbol} {bar} 窗口大小: {window_size} 从 {start} 到 {end}")
        data = self.db_market_data.query_market_data_by_symbol_bar(
            symbol, bar, start, end
        )
        if data is None:
            logging.warning(f"获取行情数据失败: {symbol} {bar} 窗口大小: {window_size} 从 {start} 到 {end}")
            return None
        else:
            if len(data) == 0:
                logging.warning(f"获取行情数据为空: {symbol} {bar} 窗口大小: {window_size} 从 {start} 到 {end}")
                return None
            else:
                if isinstance(data, list):
                    data = pd.DataFrame(data)
                elif isinstance(data, dict):
                    data = pd.DataFrame([data])
        data = self.huge_volume.detect_huge_volume(
            data=data,
            window_size=window_size,
            threshold=self.threshold,
            check_price=True,
            only_output_huge_volume=only_output_huge_volume,
            output_excel=True,
        )
        if data is not None:
            if is_update:
                for index, row in data.iterrows():
                    exist_huge_volume_data = (
                        self.db_huge_volume_data.query_data_by_symbol_bar_window_size_timestamp(
                            symbol, bar, window_size, row["timestamp"]
                        )
                    )
                    if exist_huge_volume_data is not None:
                        # remove the exist_huge_volume_data from data
                        data = data[
                            data["timestamp"] != exist_huge_volume_data["timestamp"]
                        ]
            if data is not None and len(data) > 0:
                self.db_huge_volume_data.insert_data_to_mysql(data)
            else:
                logging.warning(
                    f"此次处理巨量交易数据为空: {symbol} {bar} {start} {end}"
                )
            return data
        else:
            return None

    def batch_update_volume_spike(self, window_size: int = 50):
        for symbol in self.market_data_main.symbols:
            for bar in self.market_data_main.bars:
                self.update_volume_spike(symbol, bar, window_size)

    def update_volume_spike(self, symbol: str, bar: str, window_size: int = 50):
        try:
            self.market_data_main.update_data(symbol, bar)
            latest_huge_volume_data = self.db_huge_volume_data.query_latest_data(
                symbol, bar, window_size
            )
            if latest_huge_volume_data is None or len(latest_huge_volume_data) == 0:
                self.detect_volume_spike(symbol, bar, only_output_huge_volume=True)
                return
            else:
                earliest_date_time = latest_huge_volume_data["date_time"]
                earliest_timestamp = latest_huge_volume_data["timestamp"]
                seconds = self.get_seconds_by_bar(bar)
                earliest_timestamp = earliest_timestamp - (
                    (window_size - 1) * seconds * 1000
                )
                earliest_date_time = timestamp_to_datetime(earliest_timestamp)

                data = self.detect_volume_spike(
                    symbol=symbol,
                    bar=bar,
                    window_size=window_size,
                    start=earliest_date_time,
                    only_output_huge_volume=True,
                    is_update=True,
                )
                logging.info(
                    f"更新巨量交易数据: {symbol} {bar} 窗口大小: {window_size} 从 {earliest_date_time} 到 {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
                )
                if data is not None and len(data) > 0:
                    logging.info(f"此次更新巨量交易数据: {len(data)}条")
                else:
                    logging.info(f"此次更新巨量交易数据为空")
        except Exception as e:
            logging.error(f"更新巨量交易数据失败: {symbol} {bar} 窗口大小: {window_size} 从 {earliest_date_time} 到 {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}: {e}")

    def get_seconds_by_bar(self, bar: str):
        """
        根据bar获取秒数
        bar: 1s/1m/3m/5m/15m/30m/1H/2H/4H/6H/12H/1D/2D/3D/1W/1M/3M
        :param bar: 时间周期
        :return: 秒数
        """
        if bar == "1s":
            return 1
        elif bar == "1m":
            return 60
        elif bar == "3m":
            return 180
        elif bar == "5m":
            return 300
        elif bar == "15m":
            return 900
        elif bar == "30m":
            return 1800
        elif bar == "1H":
            return 3600
        elif bar == "2H":
            return 7200
        elif bar == "4H":
            return 14400
        elif bar == "6H":
            return 21600
        elif bar == "12H":
            return 43200
        elif bar == "1D":
            return 86400
        elif bar == "2D":
            return 172800
        elif bar == "3D":
            return 259200
        elif bar == "1W":
            return 604800
        elif bar == "1M":
            return 2592000
        elif bar == "3M":
            return 7776000
        else:
            raise ValueError(f"不支持的bar: {bar}")

    def next_periods_rise_or_fall(
        self,
        symbol: str,
        bar: str,
        window_size: int = 50,
        start: str = None,
        end: str = None,
        periods: list = [3, 5],
        output_excel: bool = False,
    ):
        if start is None:
            start = MONITOR_CONFIG.get("volume_monitor", {}).get(
                "initial_date", "2025-05-01 00:00:00"
            )
        if end is None:
            end = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        logging.info(f"开始计算巨量出现后，之后3或5个周期，上涨或下跌的比例: {symbol} {bar} 窗口大小: {window_size} 从 {start} 到 {end}")
        huge_volume_data = (
            self.db_huge_volume_data.query_huge_volume_data_by_symbol_bar_window_size(
                symbol, bar, window_size, start, end
            )
        )
        if huge_volume_data is None or len(huge_volume_data) == 0:
            logging.warning(f"获取巨量交易数据为空: {symbol} {bar} 窗口大小: {window_size} 从 {start} 到 {end}")
            return None
        else:
            if isinstance(huge_volume_data, list):
                huge_volume_data = pd.DataFrame(huge_volume_data)
            elif isinstance(huge_volume_data, dict):
                huge_volume_data = pd.DataFrame([huge_volume_data])
        market_data = self.db_market_data.query_market_data_by_symbol_bar(
            symbol, bar, start, end
        )
        if market_data is None or len(market_data) == 0:
            logging.warning(f"获取行情数据为空: {symbol} {bar} 窗口大小: {window_size} 从 {start} 到 {end}")
            return None
        else:
            if isinstance(market_data, list):
                market_data = pd.DataFrame(market_data)
            elif isinstance(market_data, dict):
                market_data = pd.DataFrame([market_data])
        if (
            huge_volume_data is not None
            and len(huge_volume_data) > 0
            and market_data is not None
            and len(market_data) > 0
        ):
            # 将huge_volume_data和market_data合并
            # market_data移除id列
            market_data = market_data.drop(columns=["id"])
            # huge_volume_data移除id列
            huge_volume_data = huge_volume_data.drop(columns=["id"])
            data = pd.merge(market_data, huge_volume_data, on="timestamp", how="left")
            # 同名的列，只是后缀为_x和_y，需要合并
            data = data.rename(
                columns={
                    "symbol_x": "symbol",
                    "bar_x": "bar",
                    "date_time_x": "date_time",
                    "open_x": "open",
                    "high_x": "high",
                    "low_x": "low",
                    "close_x": "close",
                    "volume_x": "volume",
                    "volCcy_x": "volCcy",
                    "volCCyQuote_x": "volCCyQuote",
                    "create_time_x": "create_time",
                }
            )
            data = data.drop(
                columns=[
                    "symbol_y",
                    "bar_y",
                    "date_time_y",
                    "open_y",
                    "high_y",
                    "low_y",
                    "close_y",
                    "volume_y",
                    "volCcy_y",
                    "volCCyQuote_y",
                    "create_time_y",
                ]
            )
            # 根据timestamp排序
            data = data.sort_values(by="timestamp", ascending=True)
            data["window_size"] = window_size
            data = data[
                [
                    "symbol",
                    "bar",
                    "window_size",
                    "timestamp",
                    "date_time",
                    "open",
                    "high",
                    "low",
                    "close",
                    "volume",
                    "huge_volume",
                    "volume_ratio",
                    "volume_price_spike",
                    "price_high",
                    "price_low",
                ]
            ]
            data = data.dropna()
            data = data.reset_index(drop=True)
            data, result_data = self.huge_volume.next_periods_rise_or_fall(
                data=data, window_size=window_size, periods=periods, output_excel=output_excel
            )
            return data, result_data

    def batch_next_periods_rise_or_fall(
        self,
        window_size: int = 50,
        start: str = None,
        end: str = None,
        periods: list = [3, 5],
        output_excel: bool = False,
    ):
        if start is None:
            start = MONITOR_CONFIG.get("volume_monitor", {}).get(
                "initial_date", "2025-05-01 00:00:00"
            )
        if end is None:
            end = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        data_list = []
        result_data_list = []
        for symbol in self.market_data_main.symbols:
            for bar in self.market_data_main.bars:
                data, result_data = self.next_periods_rise_or_fall(
                    symbol, bar, window_size, start, end, periods, output_excel
                )
                data_list.append(data)
                result_data_list.append(result_data)
        data = pd.concat(data_list)
        result_data = pd.concat(result_data_list)
        if output_excel:
            data = data.reset_index(drop=True)
            result_data = result_data.reset_index(drop=True)
            current_date = datetime.now().strftime("%Y%m%d%H%M%S")
            file_name = (
                f"next_periods_rise_or_fall_{current_date}.xlsx"
            )
            try:
                with pd.ExcelWriter(
                    os.path.join(self.output_folder, file_name)
                ) as writer:
                    data.to_excel(writer, sheet_name="details", index=False)
                    result_data.to_excel(
                        writer, sheet_name="next_periods_statistics", index=False
                    )
            except Exception as e:
                logging.error(f"导出Excel文件失败: {e}")
        return data, result_data


def batch_initial_detect_volume_spike(threshold: float = 2.0):
    window_sizes = WINDOW_SIZE.get("window_sizes", None)
    if window_sizes is None or not isinstance(window_sizes, list) or len(window_sizes) == 0:
        window_sizes = [50, 80, 100, 120]
    huge_volume_main = HugeVolumeMain(threshold)
    for window_size in window_sizes:
        huge_volume_main.batch_initial_detect_volume_spike(
            window_size=window_size,
            start="2025-05-01 00:00:00",
        )


def batch_update_volume_spike(threshold: float = 2.0):
    window_sizes = WINDOW_SIZE.get("window_sizes", None)
    if window_sizes is None or not isinstance(window_sizes, list) or len(window_sizes) == 0:
        window_sizes = [50, 80, 100, 120]
    huge_volume_main = HugeVolumeMain(threshold)
    for window_size in window_sizes:
        huge_volume_main.batch_update_volume_spike(window_size=window_size)


if __name__ == "__main__":
    # batch_initial_detect_volume_spike(threshold=2.0)
    batch_update_volume_spike(threshold=2.0)