blade
/
dc-ml-emea-ar
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

For AUS Prospectus, cancel visiting Vision ChatGPT when page contents without any numeric text or perhaps with messy code. 

(But should keep this logic for EMEA LUX AR, because of some special providers cases for this market documents.)
This commit is contained in:
Blade He 2025-03-18 14:15:43 -05:00
parent 6614972849
commit 0cea2e501b
3 changed files with 41 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
core/data_extraction.py
View File

@ -820,6 +820,8 @@ class DataExtraction:
exclude_data=None, exclude_data=None,
previous_page_last_fund=previous_page_fund_name previous_page_last_fund=previous_page_fund_name
) )
if extract_data is None:
continue
data_list.append(extract_data) data_list.append(extract_data)
@ -892,6 +894,8 @@ class DataExtraction:
exclude_data=page_data_list, exclude_data=page_data_list,
previous_page_last_fund=previous_page_fund_name previous_page_last_fund=previous_page_fund_name
) )
if next_page_extract_data is None:
break
next_page_data_list = next_page_extract_data.get( next_page_data_list = next_page_extract_data.get(
"extract_data", {} "extract_data", {}
).get("data", []) ).get("data", [])
@ -1076,14 +1080,18 @@ class DataExtraction:
if not exist_data_point_value_text and (not exist_numeric_value or if not exist_data_point_value_text and (not exist_numeric_value or
page_text_line_count < 3 or page_text_line_count < 3 or
len(special_code_all) > 100): len(special_code_all) > 100):
logger.info(f"Can't find numberic value in page {page_num}, apply Vision ChatGPT to extract data") if self.doc_source != "aus_prospectus":
return self.extract_data_by_page_image( logger.info(f"Can't find numberic value in page {page_num}, apply Vision ChatGPT to extract data")
page_num=page_num, return self.extract_data_by_page_image(
page_datapoints=page_datapoints, page_num=page_num,
need_exclude=False, page_datapoints=page_datapoints,
exclude_data=None, need_exclude=False,
previous_page_last_fund=previous_page_last_fund, exclude_data=None,
need_extract_text=False) previous_page_last_fund=previous_page_last_fund,
need_extract_text=False)
else:
logger.info(f"Can't find numberic value in page {page_num}, skip to extract data")
return None
else: else:
return self.extract_data_by_page_text( return self.extract_data_by_page_text(
page_num=page_num, page_num=page_num,

46
performance.ipynb
View File

@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 8, "execution_count": 1,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -34,15 +34,16 @@
" \"Benchmark\": \"benchmark_name\"\n", " \"Benchmark\": \"benchmark_name\"\n",
"}\n", "}\n",
"\n", "\n",
"path_ground_truth = r\"/data/aus_prospectus/ground_truth/46_documents_ground_truth_with_mapping.xlsx\"\n", "path_ground_truth = r\"/data/aus_prospectus/ground_truth/phase2_file/46_documents/46_documents_ground_truth_with_mapping.xlsx\"\n",
"path_generated_results = r\"C:\\Users\\rmahesh\\Downloads\\mapping_data_info_46_documents_by_text_20250317.xlsx\"\n", "# path_generated_results = r\"/data/aus_prospectus/output/mapping_data/total/mapping_data_info_46_documents_by_text_20250317.xlsx\"\n",
"path_generated_results = r\"/data/aus_prospectus/output/mapping_data/total/mapping_data_info_46_documents_by_text_20250318124530.xlsx\"\n",
"provider_mapping_file_path = r\"/data/aus_prospectus/ground_truth/phase2_file/46_documents/TopProvidersBiz.xlsx\"\n", "provider_mapping_file_path = r\"/data/aus_prospectus/ground_truth/phase2_file/46_documents/TopProvidersBiz.xlsx\"\n",
"\n" "\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 9, "execution_count": 2,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -315,7 +316,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 10, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -338,8 +339,8 @@
"buy_spread \t0.9365 \t0.9242 \t0.9491 \t0.8956 \t352 \t317 \t52 \t26 \t17 \n", "buy_spread \t0.9365 \t0.9242 \t0.9491 \t0.8956 \t352 \t317 \t52 \t26 \t17 \n",
"sell_spread \t0.9412 \t0.9329 \t0.9496 \t0.9029 \t352 \t320 \t52 \t23 \t17 \n", "sell_spread \t0.9412 \t0.9329 \t0.9496 \t0.9029 \t352 \t320 \t52 \t23 \t17 \n",
"minimum_initial_investment \t0.9737 \t0.9642 \t0.9834 \t0.9612 \t301 \t296 \t100 \t11 \t5 \n", "minimum_initial_investment \t0.9737 \t0.9642 \t0.9834 \t0.9612 \t301 \t296 \t100 \t11 \t5 \n",
"benchmark_name \t0.8092 \t0.8217 \t0.7970 \t0.8786 \t144 \t106 \t256 \t23 \t27 \n", "benchmark_name \t0.8137 \t0.8295 \t0.7985 \t0.8811 \t144 \t107 \t256 \t22 \t27 \n",
"TOTAL \t0.9367 \t0.9280 \t0.9466 \t0.9235 \t2460 \t2197 \t1608 \t183 \t132 \n", "TOTAL \t0.9372 \t0.9288 \t0.9467 \t0.9238 \t2460 \t2198 \t1608 \t182 \t132 \n",
"Total Funds Matched - 412\n", "Total Funds Matched - 412\n",
"Total Funds Not Matched - 153\n", "Total Funds Not Matched - 153\n",
"Percentage of Funds Matched - 72.9203539823009\n", "Percentage of Funds Matched - 72.9203539823009\n",
@ -354,8 +355,8 @@
"buy_spread \t0.9812 \t0.9752 \t0.9874 \t0.9636 \t162 \t157 \t2 \t4 \t2 \n", "buy_spread \t0.9812 \t0.9752 \t0.9874 \t0.9636 \t162 \t157 \t2 \t4 \t2 \n",
"sell_spread \t0.9876 \t0.9876 \t0.9876 \t0.9758 \t162 \t159 \t2 \t2 \t2 \n", "sell_spread \t0.9876 \t0.9876 \t0.9876 \t0.9758 \t162 \t159 \t2 \t2 \t2 \n",
"minimum_initial_investment \t0.9569 \t0.9531 \t0.9606 \t0.9333 \t127 \t122 \t32 \t6 \t5 \n", "minimum_initial_investment \t0.9569 \t0.9531 \t0.9606 \t0.9333 \t127 \t122 \t32 \t6 \t5 \n",
"benchmark_name \t0.7651 \t0.7808 \t0.7500 \t0.7879 \t85 \t57 \t73 \t16 \t19 \n", "benchmark_name \t0.7733 \t0.7945 \t0.7532 \t0.7939 \t85 \t58 \t73 \t15 \t19 \n",
"TOTAL \t0.9318 \t0.9258 \t0.9401 \t0.9226 \t1013 \t923 \t447 \t67 \t180 \n", "TOTAL \t0.9328 \t0.9273 \t0.9405 \t0.9232 \t1013 \t924 \t447 \t66 \t180 \n",
"Total Funds Matched - 165\n", "Total Funds Matched - 165\n",
"Total Funds Not Matched - 31\n", "Total Funds Not Matched - 31\n",
"Percentage of Funds Matched - 84.18367346938776\n", "Percentage of Funds Matched - 84.18367346938776\n",
@ -477,7 +478,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": 4,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -488,8 +489,8 @@
"{'data_point': 'performance_fee_costs', 'doc_id': 401212184, 'sec_name': 'OnePath OneAnswer Investment Portfolio - OnePath Growth Index -NE', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 401212184, 'sec_name': 'OnePath OneAnswer Investment Portfolio - OnePath Growth Index -NE', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Macquarie Income Opps', 'truth': '0.03', 'generated': '0.12', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Macquarie Income Opps', 'truth': '0.03', 'generated': '0.12', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Diversified Inc', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Diversified Inc', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Share Plus L/S', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Schroder Fixed Income', 'truth': '0', 'generated': '0.01', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Schroder Fixed Income', 'truth': '0', 'generated': '0.01', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Share Plus L/S', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Balanced Pen', 'truth': '0.24', 'generated': '0.23', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Balanced Pen', 'truth': '0.24', 'generated': '0.23', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Property Pension', 'truth': '0.67', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Property Pension', 'truth': '0.67', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Cash Pension', 'truth': '0.01', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Cash Pension', 'truth': '0.01', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
@ -508,18 +509,18 @@
"{'data_point': 'performance_fee_costs', 'doc_id': 414751292, 'sec_name': 'Platinum Japan Fund', 'truth': '0.15', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 414751292, 'sec_name': 'Platinum Japan Fund', 'truth': '0.15', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 446324179, 'sec_name': 'Lifeplan Investment Bond - Allan Gray Australian Equity Fund Class A', 'truth': '0.28', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 446324179, 'sec_name': 'Lifeplan Investment Bond - Allan Gray Australian Equity Fund Class A', 'truth': '0.28', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 446324179, 'sec_name': 'Lifeplan Investment Bond MLC Horizon 2-Capital Stable Open', 'truth': '0.05', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 446324179, 'sec_name': 'Lifeplan Investment Bond MLC Horizon 2-Capital Stable Open', 'truth': '0.05', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Small Company Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Small Company Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Value Trust -Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Value Trust -Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Australian Value Trust - Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Australian Value Trust - Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Tr AUDHdg', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Tr AUDHdg', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 550522985, 'sec_name': 'RQI Global Value – Class A', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Balanced', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Balanced', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional High Growth', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional High Growth', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Conservative', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Conservative', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Growth', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Growth', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Moderately Conservative', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Moderately Conservative', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 539261734, 'sec_name': 'ipac life choices Income Generator', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 539261734, 'sec_name': 'ipac life choices Income Generator', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 523516443, 'sec_name': 'CFS MIF-Strategic Cash', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Future Goals Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Future Goals Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Property Securities Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Property Securities Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Asian Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Asian Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
@ -529,7 +530,6 @@
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT European Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT European Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT American Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT American Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Imputation Shares Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Imputation Shares Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 523516443, 'sec_name': 'CFS MIF-Strategic Cash', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 557526129, 'sec_name': 'Fortlake Real-Income Fund', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 557526129, 'sec_name': 'Fortlake Real-Income Fund', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Core Equity Tr AUDHdg', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Core Equity Tr AUDHdg', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
@ -537,18 +537,19 @@
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Value Trust -Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Value Trust -Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Small Company Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Small Company Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 544886057, 'sec_name': 'CFS Growth Builder', 'truth': '0.01', 'generated': '0.04', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 544886057, 'sec_name': 'CFS Growth Builder', 'truth': '0.01', 'generated': '0.04', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 550522985, 'sec_name': 'RQI Global Value – Class A', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Small Company Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Small Company Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Value Trust -Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Value Trust -Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Australian Value Trust - Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Australian Value Trust - Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Tr AUDHdg', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 530101994, 'sec_name': 'Dimensional Global Core Equity Tr AUDHdg', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 550522985, 'sec_name': 'RQI Global Value – Class A', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Balanced', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Balanced', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional High Growth', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional High Growth', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Conservative', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Conservative', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Growth', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Growth', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Moderately Conservative', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 539241700, 'sec_name': 'North Professional Moderately Conservative', 'truth': '0', 'generated': '0.06', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 539261734, 'sec_name': 'ipac life choices Income Generator', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 539261734, 'sec_name': 'ipac life choices Income Generator', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 523516443, 'sec_name': 'CFS MIF-Strategic Cash', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Future Goals Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Future Goals Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Property Securities Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Property Securities Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Asian Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Asian Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
@ -558,7 +559,6 @@
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT European Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT European Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT American Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT American Share Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Imputation Shares Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 526200514, 'sec_name': 'BT Imputation Shares Retail', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 523516443, 'sec_name': 'CFS MIF-Strategic Cash', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 557526129, 'sec_name': 'Fortlake Real-Income Fund', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 557526129, 'sec_name': 'Fortlake Real-Income Fund', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Core Equity Tr AUDHdg', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Core Equity Tr AUDHdg', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Core Equity Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
@ -566,12 +566,13 @@
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Value Trust -Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Value Trust -Active ETF', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Small Company Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 531373053, 'sec_name': 'Dimensional Global Small Company Trust', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 544886057, 'sec_name': 'CFS Growth Builder', 'truth': '0.01', 'generated': '0.04', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 544886057, 'sec_name': 'CFS Growth Builder', 'truth': '0.01', 'generated': '0.04', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 550522985, 'sec_name': 'RQI Global Value – Class A', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 401212184, 'sec_name': 'ANZ OA Inv-OnePath Multi Asset Income NEF', 'truth': '0', 'generated': '0.11', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 401212184, 'sec_name': 'ANZ OA Inv-OnePath Multi Asset Income NEF', 'truth': '0', 'generated': '0.11', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 401212184, 'sec_name': 'OnePath OneAnswer Investment Portfolio - OnePath Growth Index -NE', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 401212184, 'sec_name': 'OnePath OneAnswer Investment Portfolio - OnePath Growth Index -NE', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Macquarie Income Opps', 'truth': '0.03', 'generated': '0.12', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Macquarie Income Opps', 'truth': '0.03', 'generated': '0.12', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Diversified Inc', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Diversified Inc', 'truth': '0', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Share Plus L/S', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Schroder Fixed Income', 'truth': '0', 'generated': '0.01', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Schroder Fixed Income', 'truth': '0', 'generated': '0.01', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 411062815, 'sec_name': 'Perpetual WFP-Perpetual Share Plus L/S', 'truth': '', 'generated': '0', 'error': 'Truth is null and generated is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Balanced Pen', 'truth': '0.24', 'generated': '0.23', 'error': 'Truth is not equal with generated'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Balanced Pen', 'truth': '0.24', 'generated': '0.23', 'error': 'Truth is not equal with generated'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Property Pension', 'truth': '0.67', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Property Pension', 'truth': '0.67', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
"{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Cash Pension', 'truth': '0.01', 'generated': '', 'error': 'Generated is null and truth is not null'}\n", "{'data_point': 'performance_fee_costs', 'doc_id': 412778803, 'sec_name': 'Telstra Cash Pension', 'truth': '0.01', 'generated': '', 'error': 'Generated is null and truth is not null'}\n",
@ -601,14 +602,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 13, "execution_count": 7,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"Excel file 'C:\\data\\aus_prospectus\\output\\error_analysis\\anomilities_found.xlsx' has been created successfully.\n" "Excel file '/data/aus_prospectus/output/error_analysis/anomalies_found.xlsx' has been created successfully.\n"
] ]
} }
], ],
@ -623,7 +624,8 @@
"df_sorted = df.sort_values(by=['doc_id'])\n", "df_sorted = df.sort_values(by=['doc_id'])\n",
"\n", "\n",
"# Save DataFrame to Excel file\n", "# Save DataFrame to Excel file\n",
"output_filename = r\"C:\\data\\aus_prospectus\\output\\error_analysis\\anomalies_found.xlsx\"\n", "os.makedirs(\"/data/aus_prospectus/output/error_analysis/\", exist_ok=True)\n",
"output_filename = r\"/data/aus_prospectus/output/error_analysis/anomalies_found.xlsx\"\n",
"df_sorted.to_excel(output_filename, index=False)\n", "df_sorted.to_excel(output_filename, index=False)\n",
"\n", "\n",
"print(f\"Excel file '{output_filename}' has been created successfully.\")\n" "print(f\"Excel file '{output_filename}' has been created successfully.\")\n"
@ -667,7 +669,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.12.4" "version": "3.12.6"
}, },
"orig_nbformat": 4 "orig_nbformat": 4
}, },

2
utils/biz_utils.py
View File

@ -4,7 +4,7 @@ import time
from utils.logger import logger from utils.logger import logger
from copy import deepcopy from copy import deepcopy
from traceback import print_exc from traceback import print_exc
import benchmark_names import utils.benchmark_names
total_currency_list = [ total_currency_list = [
"USD", "USD",