For AUS Prospectus, cancel visiting Vision ChatGPT when page contents without any numeric text or perhaps with messy code.
(But should keep this logic for EMEA LUX AR, because of some special providers cases for this market documents.)
This commit is contained in:
Blade He
parent
6614972849
commit
0cea2e501b
|
|
@ -820,6 +820,8 @@ class DataExtraction:
|
|||
exclude_data=None,
|
||||
previous_page_last_fund=previous_page_fund_name
|
||||
)
|
||||
if extract_data is None:
|
||||
continue
|
||||
data_list.append(extract_data)
|
||||
|
||||
|
||||
|
|
@ -892,6 +894,8 @@ class DataExtraction:
|
|||
exclude_data=page_data_list,
|
||||
previous_page_last_fund=previous_page_fund_name
|
||||
)
|
||||
if next_page_extract_data is None:
|
||||
break
|
||||
next_page_data_list = next_page_extract_data.get(
|
||||
"extract_data", {}
|
||||
).get("data", [])
|
||||
|
|
@ -1076,14 +1080,18 @@ class DataExtraction:
|
|||
if not exist_data_point_value_text and (not exist_numeric_value or
|
||||
page_text_line_count < 3 or
|
||||
len(special_code_all) > 100):
|
||||
logger.info(f"Can't find numberic value in page {page_num}, apply Vision ChatGPT to extract data")
|
||||
return self.extract_data_by_page_image(
|
||||
page_num=page_num,
|
||||
page_datapoints=page_datapoints,
|
||||
need_exclude=False,
|
||||
exclude_data=None,
|
||||
previous_page_last_fund=previous_page_last_fund,
|
||||
need_extract_text=False)
|
||||
if self.doc_source != "aus_prospectus":
|
||||
logger.info(f"Can't find numberic value in page {page_num}, apply Vision ChatGPT to extract data")
|
||||
return self.extract_data_by_page_image(
|
||||
page_num=page_num,
|
||||
page_datapoints=page_datapoints,
|
||||
need_exclude=False,
|
||||
exclude_data=None,
|
||||
previous_page_last_fund=previous_page_last_fund,
|
||||
need_extract_text=False)
|
||||
else:
|
||||
logger.info(f"Can't find numberic value in page {page_num}, skip to extract data")
|
||||
return None
|
||||
else:
|
||||
return self.extract_data_by_page_text(
|
||||
page_num=page_num,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -34,15 +34,16 @@
|
|||
" \"Benchmark\": \"benchmark_name\"\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"path_ground_truth = r\"/data/aus_prospectus/ground_truth/46_documents_ground_truth_with_mapping.xlsx\"\n",
|
||||
"path_generated_results = r\"C:\\Users\\rmahesh\\Downloads\\mapping_data_info_46_documents_by_text_20250317.xlsx\"\n",
|
||||
"path_ground_truth = r\"/data/aus_prospectus/ground_truth/phase2_file/46_documents/46_documents_ground_truth_with_mapping.xlsx\"\n",
|
||||
"# path_generated_results = r\"/data/aus_prospectus/output/mapping_data/total/mapping_data_info_46_documents_by_text_20250317.xlsx\"\n",
|
||||
"path_generated_results = r\"/data/aus_prospectus/output/mapping_data/total/mapping_data_info_46_documents_by_text_20250318124530.xlsx\"\n",
|
||||
"provider_mapping_file_path = r\"/data/aus_prospectus/ground_truth/phase2_file/46_documents/TopProvidersBiz.xlsx\"\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -315,7 +316,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -338,8 +339,8 @@
|
|||
"buy_spread \t0.9365 \t0.9242 \t0.9491 \t0.8956 \t352 \t317 \t52 \t26 \t17 \n",
|
||||
"sell_spread \t0.9412 \t0.9329 \t0.9496 \t0.9029 \t352 \t320 \t52 \t23 \t17 \n",
|
||||
"minimum_initial_investment \t0.9737 \t0.9642 \t0.9834 \t0.9612 \t301 \t296 \t100 \t11 \t5 \n",
|
||||
"benchmark_name \t0.8092 \t0.8217 \t0.7970 \t0.8786 \t144 \t106 \t256 \t23 \t27 \n",
|
||||
"TOTAL \t0.9367 \t0.9280 \t0.9466 \t0.9235 \t2460 \t2197 \t1608 \t183 \t132 \n",
|
||||
"benchmark_name \t0.8137 \t0.8295 \t0.7985 \t0.8811 \t144 \t107 \t256 \t22 \t27 \n",
|
||||
"TOTAL \t0.9372 \t0.9288 \t0.9467 \t0.9238 \t2460 \t2198 \t1608 \t182 \t132 \n",
|
||||
"Total Funds Matched - 412\n",
|
||||
"Total Funds Not Matched - 153\n",
|
||||
"Percentage of Funds Matched - 72.9203539823009\n",
|
||||
|
|
@ -354,8 +355,8 @@
|
|||
"buy_spread \t0.9812 \t0.9752 \t0.9874 \t0.9636 \t162 \t157 \t2 \t4 \t2 \n",
|
||||
"sell_spread \t0.9876 \t0.9876 \t0.9876 \t0.9758 \t162 \t159 \t2 \t2 \t2 \n",
|
||||
"minimum_initial_investment \t0.9569 \t0.9531 \t0.9606 \t0.9333 \t127 \t122 \t32 \t6 \t5 \n",
|
||||
"benchmark_name \t0.7651 \t0.7808 \t0.7500 \t0.7879 \t85 \t57 \t73 \t16 \t19 \n",
|
||||
"TOTAL \t0.9318 \t0.9258 \t0.9401 \t0.9226 \t1013 \t923 \t447 \t67 \t180 \n",
|
||||
"benchmark_name \t0.7733 \t0.7945 \t0.7532 \t0.7939 \t85 \t58 \t73 \t15 \t19 \n",
|
||||
"TOTAL \t0.9328 \t0.9273 \t0.9405 \t0.9232 \t1013 \t924 \t447 \t66 \t180 \n",
|
||||
"Total Funds Matched - 165\n",
|
||||
"Total Funds Not Matched - 31\n",
|
||||
"Percentage of Funds Matched - 84.18367346938776\n",
|
||||
|
|
@ -477,7 +478,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -488,8 +489,8 @@
|
|||
"{'data_point': 'performance_fee_costs', 'doc_id': 401212184, 'sec_name': 'OnePath OneAnswer Investment Portfolio - OnePath Growth Index -NE', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Macquarie Income Opps', 'truth': '0.03', 'generated': '0.12', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Diversified Inc', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Share Plus L/S', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Schroder Fixed Income', 'truth': '0', 'generated': '0.01', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Share Plus L/S', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Balanced Pen', 'truth': '0.24', 'generated': '0.23', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Property Pension', 'truth': '0.67', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Cash Pension', 'truth': '0.01', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
|
|
@ -508,18 +509,18 @@
|
|||
"{'data_point': 'performance_fee_costs', 'doc_id': 414751292, 'sec_name': 'Platinum Japan Fund', 'truth': '0.15', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 446324179, 'sec_name': 'Lifeplan Investment Bond - Allan Gray Australian Equity Fund Class A', 'truth': '0.28', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 446324179, 'sec_name': 'Lifeplan Investment Bond MLC Horizon 2-Capital Stable Open', 'truth': '0.05', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Small Company Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Value Trust -Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Australian Value Trust - Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Tr AUDHdg', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 550522985, 'sec_name': 'RQI Global Value – Class A', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Balanced', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional High Growth', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Conservative', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Growth', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Moderately Conservative', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 539261734, 'sec_name': 'ipac life choices Income Generator', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 523516443, 'sec_name': 'CFS MIF-Strategic Cash', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Future Goals Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Property Securities Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Asian Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
|
|
@ -529,7 +530,6 @@
|
|||
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT European Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT American Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Imputation Shares Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 523516443, 'sec_name': 'CFS MIF-Strategic Cash', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 557526129, 'sec_name': 'Fortlake Real-Income Fund', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Core Equity Tr AUDHdg', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
|
|
@ -537,18 +537,19 @@
|
|||
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Value Trust -Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Small Company Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 544886057, 'sec_name': 'CFS Growth Builder', 'truth': '0.01', 'generated': '0.04', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 550522985, 'sec_name': 'RQI Global Value – Class A', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Small Company Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Value Trust -Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Australian Value Trust - Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Tr AUDHdg', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 550522985, 'sec_name': 'RQI Global Value – Class A', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Balanced', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional High Growth', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Conservative', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Growth', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Moderately Conservative', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 539261734, 'sec_name': 'ipac life choices Income Generator', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 523516443, 'sec_name': 'CFS MIF-Strategic Cash', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Future Goals Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Property Securities Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Asian Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
|
|
@ -558,7 +559,6 @@
|
|||
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT European Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT American Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Imputation Shares Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 523516443, 'sec_name': 'CFS MIF-Strategic Cash', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 557526129, 'sec_name': 'Fortlake Real-Income Fund', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Core Equity Tr AUDHdg', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
|
|
@ -566,12 +566,13 @@
|
|||
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Value Trust -Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Small Company Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 544886057, 'sec_name': 'CFS Growth Builder', 'truth': '0.01', 'generated': '0.04', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 550522985, 'sec_name': 'RQI Global Value – Class A', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 401212184, 'sec_name': 'ANZ OA Inv-OnePath Multi Asset Income NEF', 'truth': '0', 'generated': '0.11', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 401212184, 'sec_name': 'OnePath OneAnswer Investment Portfolio - OnePath Growth Index -NE', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Macquarie Income Opps', 'truth': '0.03', 'generated': '0.12', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Diversified Inc', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Share Plus L/S', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Schroder Fixed Income', 'truth': '0', 'generated': '0.01', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Share Plus L/S', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Balanced Pen', 'truth': '0.24', 'generated': '0.23', 'error': 'Truth is not equal with generated'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Property Pension', 'truth': '0.67', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
"{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Cash Pension', 'truth': '0.01', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
|
||||
|
|
@ -601,14 +602,14 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Excel file 'C:\\data\\aus_prospectus\\output\\error_analysis\\anomilities_found.xlsx' has been created successfully.\n"
|
||||
"Excel file '/data/aus_prospectus/output/error_analysis/anomalies_found.xlsx' has been created successfully.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -623,7 +624,8 @@
|
|||
"df_sorted = df.sort_values(by=['doc_id'])\n",
|
||||
"\n",
|
||||
"# Save DataFrame to Excel file\n",
|
||||
"output_filename = r\"C:\\data\\aus_prospectus\\output\\error_analysis\\anomalies_found.xlsx\"\n",
|
||||
"os.makedirs(\"/data/aus_prospectus/output/error_analysis/\", exist_ok=True)\n",
|
||||
"output_filename = r\"/data/aus_prospectus/output/error_analysis/anomalies_found.xlsx\"\n",
|
||||
"df_sorted.to_excel(output_filename, index=False)\n",
|
||||
"\n",
|
||||
"print(f\"Excel file '{output_filename}' has been created successfully.\")\n"
|
||||
|
|
@ -667,7 +669,7 @@
|
|||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.4"
|
||||
"version": "3.12.6"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
},
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import time
|
|||
from utils.logger import logger
|
||||
from copy import deepcopy
|
||||
from traceback import print_exc
|
||||
import benchmark_names
|
||||
import utils.benchmark_names
|
||||
|
||||
total_currency_list = [
|
||||
"USD",
|
||||
|
|
|
|||
Loading…
Reference in New Issue