296 lines
12 KiB
Python
296 lines
12 KiB
Python
from pandas import DataFrame
|
||
import logging
|
||
import os
|
||
import re
|
||
import pandas as pd
|
||
from datetime import datetime
|
||
from typing import Optional, List, Dict, Any, Tuple
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
|
||
)
|
||
|
||
|
||
class HugeVolume:
|
||
def __init__(self, output_folder: str = "./output"):
|
||
self.output_folder = output_folder
|
||
os.makedirs(self.output_folder, exist_ok=True)
|
||
|
||
def _calculate_percentile_indicators(
|
||
self,
|
||
data: pd.DataFrame,
|
||
window_size: int,
|
||
percentiles: List[Tuple[float, str]] = [(0.8, "80"), (0.2, "20"), (0.9, "90"), (0.1, "10")]
|
||
) -> pd.DataFrame:
|
||
"""
|
||
计算分位数指标
|
||
:param data: 数据DataFrame
|
||
:param window_size: 窗口大小
|
||
:param percentiles: 分位数配置列表，格式为[(分位数, 名称后缀)]
|
||
:return: 包含分位数指标的DataFrame
|
||
"""
|
||
for percentile, suffix in percentiles:
|
||
# 计算分位数
|
||
data[f"close_{suffix}_percentile"] = (
|
||
data["close"].rolling(window=window_size, min_periods=1).quantile(percentile)
|
||
)
|
||
|
||
# 判断价格是否达到分位数
|
||
if suffix in ["80", "90"]:
|
||
# 高点分位数
|
||
data[f"price_{suffix}_high"] = (
|
||
data["close"] >= data[f"close_{suffix}_percentile"]
|
||
).astype(int)
|
||
else:
|
||
# 低点分位数
|
||
data[f"price_{suffix}_low"] = (
|
||
data["close"] <= data[f"close_{suffix}_percentile"]
|
||
).astype(int)
|
||
|
||
return data
|
||
|
||
def _calculate_volume_price_spikes(self, data: pd.DataFrame) -> pd.DataFrame:
|
||
"""
|
||
计算量价尖峰指标
|
||
:param data: 数据DataFrame
|
||
:return: 包含量价尖峰指标的DataFrame
|
||
"""
|
||
# 80/20量价尖峰
|
||
data["volume_80_20_price_spike"] = (
|
||
(data["huge_volume"] == 1)
|
||
& ((data["price_80_high"] == 1) | (data["price_20_low"] == 1))
|
||
).astype(int)
|
||
|
||
# 90/10量价尖峰
|
||
data["volume_90_10_price_spike"] = (
|
||
(data["huge_volume"] == 1)
|
||
& ((data["price_90_high"] == 1) | (data["price_10_low"] == 1))
|
||
).astype(int)
|
||
|
||
return data
|
||
|
||
def detect_huge_volume(
|
||
self,
|
||
data: DataFrame,
|
||
window_size: int = 50,
|
||
threshold: float = 2.0,
|
||
check_price: bool = False,
|
||
only_output_huge_volume: bool = False,
|
||
output_excel: bool = False,
|
||
) -> Optional[DataFrame]:
|
||
"""
|
||
detect_volume_spike的函数逻辑：
|
||
1. 根据window滑动行情数据
|
||
2. 每一个window的最新的volume是否高于该window的volume的均值+2倍标准差，如果满足条件，则增加一列：huge_volume，值为1
|
||
3. 如果check_price为True，则检查:
|
||
a. 每一个window的close是否处于该window的80%分位数及以上
|
||
b. 每一个window的close是否处于该window的20%分位数及以下
|
||
c. 每一个window的close是否处于该window的90%分位数及以上
|
||
d. 每一个window的close是否处于该window的10%分位数及以下
|
||
|
||
Args:
|
||
data: 包含成交量数据的DataFrame
|
||
threshold: 标准差倍数，默认为2.0（即成交量超过均值+2倍标准差）
|
||
window_size: 计算移动窗口的大小，默认50个周期
|
||
check_price: 是否检查价格处于windows内的分位数位置，默认False
|
||
only_output_huge_volume: 是否只输出巨量交易记录，默认False
|
||
output_excel: 是否输出到Excel文件，默认False
|
||
Returns:
|
||
DataFrame: 包含异常检测结果的DataFrame
|
||
"""
|
||
if data is None or len(data) == 0:
|
||
logging.warning("数据为空，无法进行成交量异常检测")
|
||
return None
|
||
|
||
if "volume" not in data.columns:
|
||
logging.error("数据中缺少volume列")
|
||
return None
|
||
|
||
# 按时间戳排序
|
||
data = data.sort_values(by="timestamp", ascending=True).copy()
|
||
|
||
# 计算移动窗口的成交量均值和标准差
|
||
data["volume_ma"] = (
|
||
data["volume"].rolling(window=window_size, min_periods=1).mean()
|
||
)
|
||
data["volume_std"] = (
|
||
data["volume"].rolling(window=window_size, min_periods=1).std()
|
||
)
|
||
|
||
# 计算成交量阈值（均值 + threshold倍标准差）
|
||
data["volume_threshold"] = data["volume_ma"] + threshold * data["volume_std"]
|
||
|
||
# 判断当前成交量是否超过阈值
|
||
data["huge_volume"] = (data["volume"] > data["volume_threshold"]).astype(int)
|
||
|
||
# 计算成交量比率
|
||
data["volume_ratio"] = data["volume"] / data["volume_ma"]
|
||
|
||
# 计算异常强度
|
||
data["spike_intensity"] = data["volume_ratio"] - 1
|
||
|
||
# 如果check_price为True，检查价格分位数
|
||
if check_price:
|
||
if "close" not in data.columns:
|
||
logging.error("数据中缺少close列，无法进行价格检查")
|
||
return data
|
||
|
||
# 计算分位数指标（80/20和90/10）
|
||
data = self._calculate_percentile_indicators(data, window_size)
|
||
|
||
# 计算量价尖峰指标
|
||
data = self._calculate_volume_price_spikes(data)
|
||
|
||
if only_output_huge_volume:
|
||
data = data[(data["huge_volume"] == 1)]
|
||
|
||
data["create_time"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
if output_excel:
|
||
# 检查数据是否为空
|
||
if len(data) == 0:
|
||
logging.warning("数据为空，无法导出Excel文件")
|
||
return data
|
||
|
||
start_date = data["date_time"].iloc[0]
|
||
end_date = data["date_time"].iloc[-1]
|
||
# remove punctuation from start_date and end_date
|
||
start_date = re.sub(r"[\:\-\s]", "", str(start_date))
|
||
end_date = re.sub(r"[\:\-\s]", "", str(end_date))
|
||
symbol = data["symbol"].iloc[0]
|
||
bar = data["bar"].iloc[0]
|
||
file_name = f"volume_spike_{symbol}_{bar}_{start_date}_{end_date}.xlsx"
|
||
try:
|
||
with pd.ExcelWriter(
|
||
os.path.join(self.output_folder, file_name)
|
||
) as writer:
|
||
data.to_excel(writer, sheet_name="volume_spike", index=False)
|
||
except Exception as e:
|
||
logging.error(f"导出Excel文件失败: {e}")
|
||
|
||
return data
|
||
|
||
def next_periods_rise_or_fall(
|
||
self,
|
||
data: pd.DataFrame,
|
||
periods: List[int] = [3, 5],
|
||
output_excel: bool = False
|
||
) -> Tuple[pd.DataFrame, pd.DataFrame]:
|
||
"""
|
||
1. 根据period_count，计算每个timestamp的下一个periods的rise_or_fall
|
||
示例，period_count=3，则计算每个timestamp的下一个3个periods的rise_or_fall
|
||
示例，data如下：
|
||
timestamp close huge_volume
|
||
1000000000 100 1
|
||
1000000001 101 0
|
||
1000000002 102 1
|
||
1000000003 103 0
|
||
1000000004 100 1
|
||
1000000005 98 0
|
||
|
||
则对于timestamp，1000000000，计算结果为：
|
||
timestamp close huge_volume next_3_close next_3_result next_5_close next_5_result
|
||
1000000000 100 1 103 rise 98 fall
|
||
因为之后第3个periods的close是103，所以next_3_result为rise
|
||
因为之后第5个periods的close是98，所以next_3_result为fall
|
||
|
||
2. 如果output_excel为True，则输出到excel
|
||
3. 新建一个列表: result，计算huge_volume为1时，之后3或5个周期，close上涨或下跌的比例
|
||
a. 计算huge_volume为1时，且price_80_high为1时的数量，如100， 并且计算next_3_result为fall的次数，如50， 然后计算fall_ratio， 如50/100=0.5
|
||
b. 计算huge_volume为1时，且price_80_high为1时的数量，如100， 并且计算next_5_result为fall的次数，如30， 然后计算fall_ratio， 如30/100=0.3
|
||
c. 计算huge_volume为1时，且price_20_low为1时的数量，如100， 并且计算next_3_result为rise的次数，如50， 然后计算rise_ratio， 如50/100=0.5
|
||
d. 计算huge_volume为1时，且price_20_low为1时的数量，如100， 并且计算next_5_result为rise的次数，如30， 然后计算rise_ratio， 如30/100=0.3
|
||
e. 同样计算90/10分位数的统计
|
||
|
||
Args:
|
||
data: 包含巨量交易数据的DataFrame
|
||
periods: 计算周期列表，默认[3, 5]
|
||
output_excel: 是否输出到Excel文件，默认False
|
||
Returns:
|
||
Tuple[pd.DataFrame, pd.DataFrame]: (处理后的数据, 统计结果)
|
||
"""
|
||
data = data.sort_values(by="timestamp", ascending=True)
|
||
data = data.reset_index(drop=True)
|
||
|
||
# 计算未来价格变化
|
||
for period in periods:
|
||
data[f"next_{period}_close"] = data["close"].shift(-period)
|
||
data[f"next_{period}_result"] = (
|
||
data[f"next_{period}_close"] / data["close"] - 1
|
||
)
|
||
data[f"next_{period}_result"] = data[f"next_{period}_result"].apply(
|
||
lambda x: (
|
||
"rise"
|
||
if pd.notna(x) and x > 0
|
||
else (
|
||
"fall"
|
||
if pd.notna(x) and x < 0
|
||
else "draw" if pd.notna(x) and x == 0 else x
|
||
)
|
||
)
|
||
)
|
||
|
||
# 过滤data, 只获取huge_volume为1，且价格处于分位数位置的行
|
||
price_conditions = []
|
||
if "price_80_high" in data.columns:
|
||
price_conditions.append(data["price_80_high"] == 1)
|
||
if "price_20_low" in data.columns:
|
||
price_conditions.append(data["price_20_low"] == 1)
|
||
if "price_90_high" in data.columns:
|
||
price_conditions.append(data["price_90_high"] == 1)
|
||
if "price_10_low" in data.columns:
|
||
price_conditions.append(data["price_10_low"] == 1)
|
||
|
||
if price_conditions:
|
||
combined_condition = data["huge_volume"] == 1
|
||
for condition in price_conditions:
|
||
combined_condition = combined_condition | condition
|
||
data = data[combined_condition]
|
||
|
||
data = data.reset_index(drop=True)
|
||
|
||
# 统计各种分位数情况的数量
|
||
price_stats = {}
|
||
for price_type in ["price_80_high", "price_20_low", "price_90_high", "price_10_low"]:
|
||
if price_type in data.columns:
|
||
price_stats[price_type] = len(data[(data["huge_volume"] == 1) & (data[price_type] == 1)])
|
||
|
||
results = []
|
||
for period in periods:
|
||
for price_type, count in price_stats.items():
|
||
if count > 0:
|
||
# 计算下跌次数
|
||
fall_count = len(
|
||
data[
|
||
(data["huge_volume"] == 1) &
|
||
(data[price_type] == 1) &
|
||
(data[f"next_{period}_result"] == "fall")
|
||
]
|
||
)
|
||
# 计算上涨次数
|
||
rise_count = len(
|
||
data[
|
||
(data["huge_volume"] == 1) &
|
||
(data[price_type] == 1) &
|
||
(data[f"next_{period}_result"] == "rise")
|
||
]
|
||
)
|
||
|
||
results.append(
|
||
{
|
||
"symbol": data["symbol"].iloc[0] if len(data) > 0 else "",
|
||
"bar": data["bar"].iloc[0] if len(data) > 0 else "",
|
||
"huge_volume": 1,
|
||
"price_type": price_type,
|
||
"next_period": period,
|
||
"fall_count": fall_count,
|
||
"rise_count": rise_count,
|
||
"fall_ratio": fall_count / count,
|
||
"rise_ratio": rise_count / count,
|
||
"total_count": count,
|
||
}
|
||
)
|
||
|
||
result_data = pd.DataFrame(results)
|
||
return data, result_data
|