379 lines
14 KiB
Python
379 lines
14 KiB
Python
import pandas as pd
|
||
import logging
|
||
from core.db_manager import DBData
|
||
from core.utils import check_date_time_format, datetime_to_timestamp
|
||
|
||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s")
|
||
|
||
|
||
class DBHugeVolumeData:
|
||
def __init__(
|
||
self,
|
||
db_url: str
|
||
):
|
||
self.db_url = db_url
|
||
self.table_name = "crypto_huge_volume"
|
||
self.columns = [
|
||
"symbol",
|
||
"bar",
|
||
"timestamp",
|
||
"date_time",
|
||
"open",
|
||
"high",
|
||
"low",
|
||
"close",
|
||
"volume",
|
||
"volCcy",
|
||
"volCCyQuote",
|
||
"volume_ma",
|
||
"volume_std",
|
||
"volume_threshold",
|
||
"huge_volume",
|
||
"volume_ratio",
|
||
"spike_intensity",
|
||
"close_80_percentile",
|
||
"close_20_percentile",
|
||
"price_high",
|
||
"price_low",
|
||
"volume_price_spike",
|
||
"create_time",
|
||
]
|
||
self.db_manager = DBData(db_url, self.table_name, self.columns)
|
||
|
||
def insert_data_to_mysql(self, df: pd.DataFrame):
|
||
"""
|
||
将巨量交易数据保存到MySQL的crypto_huge_volume表
|
||
速度：⭐⭐⭐⭐⭐ 最快
|
||
内存：⭐⭐⭐⭐ 中等
|
||
适用场景：中小数据量（<10万条）
|
||
:param df: 巨量交易数据DataFrame
|
||
"""
|
||
if df is None or df.empty:
|
||
logging.warning("DataFrame为空，无需写入数据库。")
|
||
return
|
||
|
||
self.db_manager.insert_data_to_mysql(df)
|
||
|
||
def insert_data_to_mysql_fast(self, df: pd.DataFrame):
|
||
"""
|
||
快速插入巨量交易数据（方案2：使用executemany批量插入）
|
||
速度：⭐⭐⭐⭐ 很快
|
||
内存：⭐⭐⭐⭐⭐ 低
|
||
适用场景：中等数据量
|
||
:param df: 巨量交易数据DataFrame
|
||
"""
|
||
if df is None or df.empty:
|
||
logging.warning("DataFrame为空，无需写入数据库。")
|
||
return
|
||
|
||
self.db_manager.insert_data_to_mysql_fast(df)
|
||
|
||
def insert_data_to_mysql_chunk(self, df: pd.DataFrame, chunk_size: int = 1000):
|
||
"""
|
||
分块插入巨量交易数据（方案3：适合大数据量）
|
||
速度：⭐⭐⭐ 中等
|
||
内存：⭐⭐⭐⭐⭐ 最低
|
||
适用场景：大数据量（>10万条）
|
||
:param df: 巨量交易数据DataFrame
|
||
:param chunk_size: 分块大小
|
||
"""
|
||
if df is None or df.empty:
|
||
logging.warning("DataFrame为空，无需写入数据库。")
|
||
return
|
||
|
||
self.db_manager.insert_data_to_mysql_chunk(df, chunk_size)
|
||
|
||
def insert_data_to_mysql_simple(self, df: pd.DataFrame):
|
||
"""
|
||
简单插入巨量交易数据（方案4：直接使用to_sql，忽略重复）
|
||
速度：⭐⭐⭐⭐⭐ 最快
|
||
内存：⭐⭐⭐⭐ 中等
|
||
注意：会抛出重复键错误，需要额外处理
|
||
"""
|
||
if df is None or df.empty:
|
||
logging.warning("DataFrame为空，无需写入数据库。")
|
||
return
|
||
|
||
self.db_manager.insert_data_to_mysql_simple(df)
|
||
|
||
def query_latest_data(self, symbol: str, bar: str):
|
||
"""
|
||
查询最新巨量交易数据
|
||
:param symbol: 交易对
|
||
:param bar: K线周期
|
||
"""
|
||
sql = """
|
||
SELECT * FROM crypto_huge_volume
|
||
WHERE symbol = :symbol AND bar = :bar
|
||
ORDER BY timestamp DESC
|
||
LIMIT 1
|
||
"""
|
||
condition_dict = {"symbol": symbol, "bar": bar}
|
||
return self.db_manager.query_data(sql, condition_dict, return_multi=False)
|
||
|
||
def query_data_by_symbol_bar_timestamp(self, symbol: str, bar: str, timestamp: int):
|
||
"""
|
||
根据交易对、K线周期和时间戳查询巨量交易数据
|
||
:param symbol: 交易对
|
||
:param bar: K线周期
|
||
:param timestamp: 时间戳
|
||
"""
|
||
sql = """
|
||
SELECT * FROM crypto_huge_volume
|
||
WHERE symbol = :symbol AND bar = :bar AND timestamp = :timestamp
|
||
"""
|
||
condition_dict = {"symbol": symbol, "bar": bar, "timestamp": timestamp}
|
||
return self.db_manager.query_data(sql, condition_dict, return_multi=False)
|
||
|
||
def query_huge_volume_data_by_symbol_bar(self, symbol: str, bar: str, start: str = None, end: str = None):
|
||
"""
|
||
根据交易对和K线周期查询巨量交易数据
|
||
:param symbol: 交易对
|
||
:param bar: K线周期
|
||
:param start: 开始时间
|
||
:param end: 结束时间
|
||
"""
|
||
if start is None or end is None:
|
||
sql = """
|
||
SELECT * FROM crypto_huge_volume
|
||
WHERE symbol = :symbol AND bar = :bar
|
||
ORDER BY timestamp ASC
|
||
"""
|
||
condition_dict = {"symbol": symbol, "bar": bar}
|
||
else:
|
||
if start is not None:
|
||
if isinstance(start, str):
|
||
if start.isdigit():
|
||
start = int(start)
|
||
else:
|
||
start = check_date_time_format(start)
|
||
# 判断是否是日期时间格式
|
||
if start is None:
|
||
logging.warning(f"日期时间格式错误: {start}")
|
||
return None
|
||
start = datetime_to_timestamp(start)
|
||
if end is not None:
|
||
if isinstance(end, str):
|
||
if end.isdigit():
|
||
end = int(end)
|
||
else:
|
||
end = check_date_time_format(end)
|
||
if end is None:
|
||
logging.warning(f"日期时间格式错误: {end}")
|
||
return None
|
||
end = datetime_to_timestamp(end)
|
||
if start is not None and end is not None:
|
||
if start > end:
|
||
start, end = end, start
|
||
sql = """
|
||
SELECT * FROM crypto_huge_volume
|
||
WHERE symbol = :symbol AND bar = :bar AND timestamp BETWEEN :start AND :end
|
||
ORDER BY timestamp ASC
|
||
"""
|
||
condition_dict = {"symbol": symbol, "bar": bar, "start": start, "end": end}
|
||
elif start is not None:
|
||
sql = """
|
||
SELECT * FROM crypto_huge_volume
|
||
WHERE symbol = :symbol AND bar = :bar AND timestamp >= :start
|
||
ORDER BY timestamp ASC
|
||
"""
|
||
condition_dict = {"symbol": symbol, "bar": bar, "start": start}
|
||
elif end is not None:
|
||
sql = """
|
||
SELECT * FROM crypto_huge_volume
|
||
WHERE symbol = :symbol AND bar = :bar AND timestamp <= :end
|
||
ORDER BY timestamp ASC
|
||
"""
|
||
condition_dict = {"symbol": symbol, "bar": bar, "end": end}
|
||
return self.db_manager.query_data(sql, condition_dict, return_multi=True)
|
||
|
||
def query_huge_volume_records(self, symbol: str = None, bar: str = None, start: str = None, end: str = None):
|
||
"""
|
||
查询巨量交易记录（只返回huge_volume=1的记录）
|
||
:param symbol: 交易对
|
||
:param bar: K线周期
|
||
:param start: 开始时间
|
||
:param end: 结束时间
|
||
"""
|
||
conditions = ["huge_volume = 1"]
|
||
condition_dict = {}
|
||
|
||
if symbol:
|
||
conditions.append("symbol = :symbol")
|
||
condition_dict["symbol"] = symbol
|
||
if bar:
|
||
conditions.append("bar = :bar")
|
||
condition_dict["bar"] = bar
|
||
if start:
|
||
if isinstance(start, str):
|
||
if start.isdigit():
|
||
start = int(start)
|
||
else:
|
||
start = check_date_time_format(start)
|
||
if start is None:
|
||
logging.warning(f"日期时间格式错误: {start}")
|
||
return None
|
||
start = datetime_to_timestamp(start)
|
||
conditions.append("timestamp >= :start")
|
||
condition_dict["start"] = start
|
||
if end:
|
||
if isinstance(end, str):
|
||
if end.isdigit():
|
||
end = int(end)
|
||
else:
|
||
end = check_date_time_format(end)
|
||
if end is None:
|
||
logging.warning(f"日期时间格式错误: {end}")
|
||
return None
|
||
end = datetime_to_timestamp(end)
|
||
conditions.append("timestamp <= :end")
|
||
condition_dict["end"] = end
|
||
|
||
where_clause = " AND ".join(conditions)
|
||
sql = f"""
|
||
SELECT * FROM crypto_huge_volume
|
||
WHERE {where_clause}
|
||
ORDER BY timestamp DESC
|
||
"""
|
||
|
||
return self.db_manager.query_data(sql, condition_dict, return_multi=True)
|
||
|
||
def query_volume_price_spike_records(self, symbol: str = None, bar: str = None, start: str = None, end: str = None):
|
||
"""
|
||
查询量价尖峰记录（只返回volume_price_spike=1的记录）
|
||
:param symbol: 交易对
|
||
:param bar: K线周期
|
||
:param start: 开始时间
|
||
:param end: 结束时间
|
||
"""
|
||
conditions = ["volume_price_spike = 1"]
|
||
condition_dict = {}
|
||
|
||
if symbol:
|
||
conditions.append("symbol = :symbol")
|
||
condition_dict["symbol"] = symbol
|
||
if bar:
|
||
conditions.append("bar = :bar")
|
||
condition_dict["bar"] = bar
|
||
if start:
|
||
if isinstance(start, str):
|
||
if start.isdigit():
|
||
start = int(start)
|
||
else:
|
||
start = check_date_time_format(start)
|
||
if start is None:
|
||
logging.warning(f"日期时间格式错误: {start}")
|
||
return None
|
||
start = datetime_to_timestamp(start)
|
||
conditions.append("timestamp >= :start")
|
||
condition_dict["start"] = start
|
||
if end:
|
||
if isinstance(end, str):
|
||
if end.isdigit():
|
||
end = int(end)
|
||
else:
|
||
end = check_date_time_format(end)
|
||
if end is None:
|
||
logging.warning(f"日期时间格式错误: {end}")
|
||
return None
|
||
end = datetime_to_timestamp(end)
|
||
conditions.append("timestamp <= :end")
|
||
condition_dict["end"] = end
|
||
|
||
where_clause = " AND ".join(conditions)
|
||
sql = f"""
|
||
SELECT * FROM crypto_huge_volume
|
||
WHERE {where_clause}
|
||
ORDER BY timestamp DESC
|
||
"""
|
||
|
||
return self.db_manager.query_data(sql, condition_dict, return_multi=True)
|
||
|
||
def get_statistics_summary(self, symbol: str = None, bar: str = None, start: str = None, end: str = None):
|
||
"""
|
||
获取巨量交易统计摘要
|
||
:param symbol: 交易对
|
||
:param bar: K线周期
|
||
:param start: 开始时间
|
||
:param end: 结束时间
|
||
"""
|
||
conditions = []
|
||
condition_dict = {}
|
||
|
||
if symbol:
|
||
conditions.append("symbol = :symbol")
|
||
condition_dict["symbol"] = symbol
|
||
if bar:
|
||
conditions.append("bar = :bar")
|
||
condition_dict["bar"] = bar
|
||
if start:
|
||
if isinstance(start, str):
|
||
if start.isdigit():
|
||
start = int(start)
|
||
else:
|
||
start = check_date_time_format(start)
|
||
if start is None:
|
||
logging.warning(f"日期时间格式错误: {start}")
|
||
return None
|
||
start = datetime_to_timestamp(start)
|
||
conditions.append("timestamp >= :start")
|
||
condition_dict["start"] = start
|
||
if end:
|
||
if isinstance(end, str):
|
||
if end.isdigit():
|
||
end = int(end)
|
||
else:
|
||
end = check_date_time_format(end)
|
||
if end is None:
|
||
logging.warning(f"日期时间格式错误: {end}")
|
||
return None
|
||
end = datetime_to_timestamp(end)
|
||
conditions.append("timestamp <= :end")
|
||
condition_dict["end"] = end
|
||
|
||
where_clause = " AND ".join(conditions) if conditions else "1=1"
|
||
sql = f"""
|
||
SELECT
|
||
COUNT(*) as total_records,
|
||
SUM(huge_volume) as huge_volume_count,
|
||
SUM(volume_price_spike) as volume_price_spike_count,
|
||
SUM(price_high) as price_high_count,
|
||
SUM(price_low) as price_low_count,
|
||
AVG(volume_ratio) as avg_volume_ratio,
|
||
MAX(volume_ratio) as max_volume_ratio,
|
||
AVG(spike_intensity) as avg_spike_intensity,
|
||
MAX(spike_intensity) as max_spike_intensity
|
||
FROM crypto_huge_volume
|
||
WHERE {where_clause}
|
||
"""
|
||
|
||
return self.db_manager.query_data(sql, condition_dict, return_multi=False)
|
||
|
||
def get_top_volume_spikes(self, symbol: str = None, bar: str = None, limit: int = 10):
|
||
"""
|
||
获取成交量尖峰最高的记录
|
||
:param symbol: 交易对
|
||
:param bar: K线周期
|
||
:param limit: 返回记录数量
|
||
"""
|
||
conditions = ["huge_volume = 1"]
|
||
condition_dict = {}
|
||
|
||
if symbol:
|
||
conditions.append("symbol = :symbol")
|
||
condition_dict["symbol"] = symbol
|
||
if bar:
|
||
conditions.append("bar = :bar")
|
||
condition_dict["bar"] = bar
|
||
|
||
where_clause = " AND ".join(conditions)
|
||
sql = f"""
|
||
SELECT * FROM crypto_huge_volume
|
||
WHERE {where_clause}
|
||
ORDER BY volume_ratio DESC
|
||
LIMIT :limit
|
||
"""
|
||
condition_dict["limit"] = limit
|
||
|
||
return self.db_manager.query_data(sql, condition_dict, return_multi=True)
|