optimize for management fees

configuration/aus_prospectus/datapoint_keyword.json

@@ -1,7 +1,7 @@
 {
   "total_annual_dollar_based_charges": {"english": ["total annual dollar based charges", "total annual dollar based charges ($)","total annual dollar"]},
-  "management_fee_and_costs": {"english": ["management fees and cost", "management fees and costs", "investment fees and costs", "Management costs", "investment fee and costs"]},
+  "management_fee_and_costs": {"english": ["management fees and cost", "management fees and costs", "investment fees and costs", "Management costs", "investment fee and costs", "Investment fees"]},
-  "management_fee": {"english": ["management fee", "management fees","investment management fees","management fees and cost", "investment option management costs", "investment option management costs1", "investment fees and costs", "investment fee and costs", "Management costs"]},
+  "management_fee": {"english": ["management fee", "management fees","investment management fees","management fees and cost", "investment option management costs", "investment option management costs1", "investment fees and costs", "investment fee and costs", "Management costs", "Investment fees"]},
   "performance_fee": {"english": ["performance fee", "performance fees"]},
   "performance_fee_costs": {"english": ["performance fee costs", "performance fees costs"]},
   "buy_spread": {"english": ["buy-spread", "buy spread", "buy/sell spreads", "BUY-SELL SPREAD"]},

configuration/aus_prospectus/datapoint_reported_name.json

@@ -1,7 +1,7 @@
 {
     "total_annual_dollar_based_charges": {"english": ["total annual dollar based charges", "total annual dollar based charges ($)","total annual dollar"]},
-    "management_fee_and_costs": {"english": ["management fees and cost", "management fees and costs", "management fee and cost", "Plus other investment fees and costs", "Management costs", "investment fees and costs", "investment fee and cost"]},
+    "management_fee_and_costs": {"english": ["management fees and cost", "management fees and costs", "management fee and cost", "Plus other investment fees and costs", "Management costs", "investment fees and costs", "investment fee and cost", "Investment fees"]},
-    "management_fee": {"english": ["management fee", "management fees","investment management fees","management fees and cost", "investment option management costs", "investment option management costs1", "Plus other investment fees and costs", "Management costs", "investment fees and costs", "investment fee and cost"]},
+    "management_fee": {"english": ["management fee", "management fees","investment management fees","management fees and cost", "investment option management costs", "investment option management costs1", "Plus other investment fees and costs", "Management costs", "investment fees and costs", "investment fee and cost", "Investment fees"]},
     "performance_fee": {"english": ["performance fee", "performance fees"]},
     "performance_fee_costs": {"english": ["performance fee costs", "performance fees costs"]},
     "buy_spread": {"english": ["buy-spread", "buy spread", "buy/sell spreads", "BUY-SELL SPREAD"]},

core/data_extraction.py

@@ -515,7 +515,6 @@ class DataExtraction:
                             break
         return data_list
         
-    
     def supplement_minimum_initial_investment(self, data_list: list):
         """
         Minimum initial investment should be same as from every fund/ share class in the same document.
@@ -576,7 +575,7 @@ class DataExtraction:
         previous_page_datapoints = []
         previous_page_fund_name = None
         for page_num, page_text in self.page_text_dict.items():
-            # if page_num < 75:
+            # if page_num != 16:
             #     continue
             if page_num in handled_page_num_list:
                 continue

instructions/aus_prospectus/data_extraction_prompts_config.json

@@ -34,9 +34,6 @@
 			"---- Example End ----",
 			"Correct fund name: AXA World Funds - ACT Emerging Markets Short Duration Bonds Low Carbon",
 			"\n",
-			"- 3. Only extract the latest data from context:",
-			"If with multiple data values in same row, please extract the latest.",
-			"\n",
 			"d. Some table format, the fund name is in the end of row, please extract the fund name from the end of row.",
 			"---Example Start---",
 			"\nTotal\nTransaction Costs\nPerformance Fees\nManagement fees and costs\nIndirect Fee\nManagement fees\nMLC diversified investment\noption\n1.49% p.a.\n0.01% p.a.\n0.06% p.a.\n0.07% p.a.\n1.35% p.a.\nMLC Horizon 2\nIncome Portfolio\n",
@@ -51,6 +48,10 @@
 			"---Example End---",
 			"Correct fund name: MLC Horizon 2 Income Portfolio",
 			"Correct share name: MLC Horizon 2 Income Portfolio",
+			"\n",
+			"- 3. Only extract the latest data from context:",
+			"If with multiple data values in same row, please extract the latest.",
+			"\n",
 			"- 4. Reported names:",
 			"Only output the values which with significant reported names.",
 			"- Multiple data columns with same reported name but different post-fix:",
@@ -87,7 +88,7 @@
 			"date_of_last_hwm_reset": "Date of last hwm reset is share class level data.",
 			"date_of_last_performance_fee_restructure": "Date of last performance fee restructure is share class level data.",
 			"high_water_mark_type": "High water mark type is share class level data.",
-			"minimum_initial_investment": "Minimum initial investment is share class level data.",
+			"minimum_initial_investment": "Minimum initial investment is fund level data.",
 			"recoverable_expenses": "Recoverable expenses is share class level data.",
 			"indirect_costs": "Indirect costs is share class level data."
 		},
@@ -163,33 +164,82 @@
 				"The output should be:",
 				"{\"data\": [{\"fund name\": \"SPDR World\", \"share name\": \"SPDR World\", \"management_fee_and_costs\": 0.18, \"management_fee\": 0.18}, {\"fund name\": \"SPDR World (Hedged)\", \"share name\": \"SPDR World (Hedged)\", \"management_fee_and_costs\": 0.21, \"management_fee\": 0.21}]}",
 				"\n",
-				"D. With table header: \"Management Fees and costs (A)\" and \"(A)+(B) + (C) = (D) Total Fees and Costs\", please only focus the values under \"Management Fees and costs (A)\"",
+				"D. With table header: \"Management Fees and costs (A)\" which span 3 sub-columns.\n Please get the 1st column number and 3rd column number from the sub-columns values,",
-				"Please get the first \"Entry Fee Option\" and \"Estimated Other investment costs\" sub-columns values, and sum as the management_fee_and_costs and management_fee value, ignore other columns values \n",
+				"and sum them as the management_fee_and_costs and management_fee value, ignore other columns values.",
+				"Attention:",
+				"1. For this case, management_fee is equal with management_fee_and_costs.",
+				"2. There are only two decimal places for each number.",
+				"3. Please totally ignore the message in context: \"(A)+(B) + (C) = (D) Total Fees and Costs\"",
+				"4. The values need to sum the the 1st number and the 3rd number.",
+				"Example to calculation pipeline for this case:",
+				"1.54 2.390.13 0.410.00 2.08 2.93",
+				"a. split the number by regex as \\d\\.\\d\\d format,",
+				"and get 7 numbers: 1.54 2.39 0.13 0.41 0.00 2.08 2.93",
+				"b. Sum the 1st number and the 3rd number only: 1.54 + 0.13 = 1.67",
+				"c. Attention: please ignore other numbers, especially the 4th number: 0.41, the sum **is not** 1.54 + 0.13 + 0.41 = 2.08!!",
+				"d. management_fee_and_costs is equal with management_fee, both of them are 1.54 + 0.13 = 1.67 for this case.",
+				"More examples:",
 				"---Example 1 Start---",
 				"Management Fees and costs (A) \nOngoing Fee (% p.a.) ‡‡ (A)+(B) + (C) = (D) Total Fees and Costs \nInvestment fund \nEstimated Other \nEstimated \nEstimated \nEntry Fee \nNil Entry \nEntry Fee \noption* \nNil Entry \nFee option \n† \ninvestment costs \nPerformance \nfees (B) \nTransaction \ncosts (C) \noption \nFee option † \nOnePath International Shares \nIndex (Hedged) \n0.47 1.320.02 0.000.00 0.49 1.32\n",
 				"---Example 1 End---",
-				"For this case, the first \"Entry Fee Option\" value is 0.47, the first \"Estimated Other investment costs\" value is 0.02, the sum is 0.49, so the output should be:",
+				"For this case, the 1st number is 0.47, the 3rd number value is 0.02, the sum is 0.49, management_fee is equal with management_fee_and_costs, so the output should be:",
 				"{\"data\": [{\"fund name\": \"OnePath International Shares Index (Hedged)\", \"share name\": \"OnePath International Shares Index (Hedged)\", \"management_fee_and_costs\": 0.49, \"management_fee\": 0.49}]}",
 				"---Example 2 Start---",
 				"Management Fees and costs (A) \nOngoing Fee (% p.a.) ‡‡ (A)+(B) + (C) = (D) Total Fees and Costs \nInvestment fund \nEstimated Other \nEstimated \nEstimated \nEntry Fee \nNil Entry \nEntry Fee \noption* \nNil Entry \nFee option \n† \ninvestment costs \nPerformance \nfees (B) \nTransaction \ncosts (C) \noption \nFee option † \nPendal Concentrated Global \nShares Hedged II \n1.44 2.290.00 0.000.04 1.48 2.33\n",
 				"---Example 2 End---",
-				"For this case, the first \"Entry Fee Option\" value is 1.44, the first \"Estimated Other investment costs\" value is 0.00, the sum is 1.44, so the output should be:",
+				"For this case, the 1st number is 1.44, the 3rd number is 0.00, the sum is 1.44, management_fee is equal with management_fee_and_costs, so the output should be:",
 				"{\"data\": [{\"fund name\": \"Pendal Concentrated Global Shares Hedged II\", \"share name\": \"Pendal Concentrated Global Shares Hedged II\", \"management_fee_and_costs\": 1.44, \"management_fee\": 1.44}]}",
 				"\n",
-				"E. If the management fee/ management fee and costs is with the range, please ignore and output empty.",
+				"E. If the table with columns:",
+				"\"Administration fees (% pa)\", \"Investment fees (% pa)\" and \"Estimated other investment costs (% pa)\"",
+				"The administration_fees is \"Administration fees (% pa)\"",
+				"The management_fee is \"Investment fees (% pa)\".",
+				"The management_fee_and_costs is \"Investment fees (% pa)\" + \"Estimated other investment costs (% pa)\".",
+				"---Example 1 Start---",
+				"Investment \noption \nAdministration fees and \nestimated administration costs \nInvestment fees and estimated \ninvestment costs \nEstimated investment \ncosts \nAdministration \nfees \n(% pa) \nInvestment \nfees \n(% pa) \n2 \nEstimated \ntotal \nongoing \nEstimated \nadministration \ncosts \n(% pa) \n1 \nEstimated \nperformance \nfees \n(% pa) \n3 \nEstimated \ntransaction \ncosts \n(% pa) \n5 \nEstimated \nother \ninvestment \ncosts \n(% pa) \n4 \nannual \nfees and \ncosts \n(% pa) \nCash \nPerpetual Cash \n0.10% \n0.00% \n0.00% \nn/a \n0.00% \n0.02% \n0.12% \nFixed income and credit \nBentham Global \nIncome \n0.25% \n0.00% \n0.67% \nn/a \n0.00% \n0.05% \n0.97% \nProperty and infrastructure \nLazard Global \nListed \nInfrastructure \n0.25% \n0.00% \n0.80% \nn/a \n0.08% \n0.09% \n1.22% \n",
+				"---Example 1 End---",
+				"For this case, although the table header is with disorder issue during PDF contents extraction issue.", 
+				"But the data points numbers order in data row (for example: 0.25% \n0.00% \n0.80% \nn/a \n0.08% \n0.09% \n1.22% \n) is correct as initial table structure.",
+				"Please pay attention below information",
+				"Assume the column sequence number is from 1.", 
+				"\"Administration fees (% pa)\" values are as the column 1 numbers, \"Investment fees (% pa)\" values are as the column 3 numbers, \"Estimated other investment costs (% pa)\" values are as the column 5 numbers.",
+				"For fund: Lazard Global Listed Infrastructure, the administration_fees should be the column 1 number: 0.25, the management_fee should be the column 3 number: 0.8, the management_fee_and_costs should be 0.88 = 0.8(the column 3 number) + 0.08 (the column 5 number)",
+				"Therefore, the output should be:",
+				"{\"data\": [{\"fund name\": \"Perpetual Cash\", \"share name\": \"Perpetual Cash\", \"management_fee_and_costs\": 0, \"management_fee\": 0, \"administration_fees\": 0.10}, {\"fund name\": \"Bentham Global Income\", \"share name\": \"Bentham Global Income\", \"management_fee_and_costs\": 0.67, \"management_fee\": 0, \"administration_fees\": 0.25}]}, {\"fund name\": \"Lazard Global Listed Infrastructure\", \"share name\": \"Lazard Global Listed Infrastructure\", \"management_fee_and_costs\": 0.88, \"management_fee\": 0.08, \"administration_fees\": 0.25}",
+				"F. If the management fee/ management fee and costs is with the range, e.g. 0.05% to 1.00%, please ignore and output empty.",
 				"---Example 1 Start---",
 				"Fees and costs summary \n\nLifeplan Investment Bond \n\nType of fee or cost Amount How and when paid \nOngoing annual fees and costs \nManagement fees and costs 6, 7 \n• \nadministration fee 1,2 of 0.60% p.a. gross of tax \ndeductions (or 0.42% p.a. net of tax deductions) \n7 , \nThe fees and costs for managing \nyour investment \n• \nless \nThe administration fee is calculated and accrued \ndaily and paid monthly in arrears from the \ninvestment option. The administration fee can be \nnegotiated with wholesale clients. 2 \nadministration fee rebate for balances of \n$500,000 or more (refer to ‘Administration fee \nrebate’ section), \nFor the Lifeplan Capital Guaranteed investment \noption \nplus \n• \nThe investment option management costs for each \ninvestment option are shown ‘in the ‘Management \nfees and costs’ section below. \ninvestment option management cost 3 charged \nby the fund managers to manage the underlying \nportfolio estimated between 0.26% and 1.82% p.a. \nfor the previous financial year for the investment \noption. 8 \n",
 				"---Example 1 End---",
 				"The relevant values: 0.26 and 1.82, are in the range, so the output should be:",
 				"{\"data\": []}",
+				"---Example 2 Start---",
+				"Investment \nfees and costs \n2 \n0.07% to 1.00% \nof assets p.a. \ndepending on \nthe investment \noption \n",
+				"---Example 2 End---",
+				"The relevant values: 0.07 and 1.00, are in the range, should ignore, so the output should be:",
+				"{\"data\": []}",
 				"\n",
-				"F. If the management fee and costs including the performance fee, please ignore the performance fee value, just output the management fee and costs value.",
+				"G. If the management fee and costs value including the performance fee, please exclude or subtract the performance fee value, just output the management fee and costs value.",
 				"---Example 1 Start---",
 				"Fees and costs for \nyour investment options \n\nAdministration fees and costs apply in addition to the fees and costs shown in this table. Please refer to the PDS and Fee Brochure for \nfurther information about fees and costs, including how the figures shown below are calculated. \n\nThe investment fees and \ncosts are made up of \nPerformance \nfee \nPlus \nother \ninvestment \nfees and \ncosts \nEquals \ninvestment \nfees and \ncosts \nTransaction \ncosts (net) \nBuy-sell \nspreads \nTransaction \ncosts \n(gross) 1 \n% pa \n% pa \n% pa \nEntry %/ \nExit % \n% pa \nMLC multi-asset portfolios\nMLC Inflation Plus\nConservative Portfolio\nSuper & Pension \npre-retirement phase \n0.18 \n0.77 \n0.95 \n0.01 \n0.10 / 0.10 \n0.09 \nRetirement Phase \n0.18 \n0.77 \n0.95 \n0.01 \n0.10 / 0.10 \n0.09 \n",
 				"---Example 1 End---",
 				"The column: \"Equals investment fees and costs\" is the sum of \"Performance fee\" and \"Plus other investment fees and costs\", we should ignore the \"Performance fee\" value, just output the \"Plus other investment fees and costs\" value.",
 				"The \"Plus other investment fees and costs\" could be the values for both of \"management fee\" and \"management fee and costs\", so the output should be:",
-				"{\"data\": [{\"fund name\": \"MLC Inflation Plus Conservative Portfolio\", \"share name\": \"Super & Pension pre-retirement phase\", \"management_fee_and_costs\": 0.77, \"management_fee\": 0.77, \"performance_fee\": 0.18, \"buy_spread\": 0.1, \"sell_spread\": 0.1}, {\"fund name\": \"MLC Inflation Plus Conservative Portfolio\", \"share name\": \"Retirement Phase\", \"management_fee_and_costs\": 0.77, \"management_fee\": 0.77, \"performance_fee\": 0.18, \"buy_spread\": 0.1, \"sell_spread\": 0.1}]}"
+				"{\"data\": [{\"fund name\": \"MLC Inflation Plus Conservative Portfolio\", \"share name\": \"Super & Pension pre-retirement phase\", \"management_fee_and_costs\": 0.77, \"management_fee\": 0.77, \"performance_fee\": 0.18, \"buy_spread\": 0.1, \"sell_spread\": 0.1}, {\"fund name\": \"MLC Inflation Plus Conservative Portfolio\", \"share name\": \"Retirement Phase\", \"management_fee_and_costs\": 0.77, \"management_fee\": 0.77, \"performance_fee\": 0.18, \"buy_spread\": 0.1, \"sell_spread\": 0.1}]}",
+				"---Example 2 Start---",
+				"MANAGEMENT COSTS AND TRANSACTION COSTS \n\nOption name Management costs \nEstimated \nperformance \nfee (pa) 1 \nTotal management\ncosts (including\nestimated performance\nfee) pa\nTransaction costs \nper transaction (%) \nMULTI-MANAGER MULTI-SECTOR (These investment options are located in the ‘Investment Options Menu’ on pages 18 to 19.) \nFirstChoice Wholesale Defensive 0.85% 0.85% 0.15\nFirstChoice Wholesale Conservative 0.90% 0.02%1 0.92% 1 0.15 \n",
+				"---Example 2 End---",
+				"The column: \"Total management costs (including estimated performance fee) pa\" is the sum of \"Management costs\" and \"Estimated performance fee (pa)\", we should ignore the \"Estimated performance fee (pa)\" value, just output the \"Management costs\" value.",
+				"Both of management_fee and management_fee_and_costs are the values for \"Management costs\", so the output should be:",
+				"{\"data\": [{\"fund name\": \"FirstChoice Wholesale Defensive\", \"share name\": \"FirstChoice Wholesale Defensive\", \"management_fee_and_costs\": 0.85, \"management_fee\": 0.85}, {\"fund name\": \"FirstChoice Wholesale Conservative\", \"share name\": \"FirstChoice Wholesale Conservative\", \"management_fee_and_costs\": 0.9, \"management_fee\": 0.9, \"performance_fee\": 0.02}]}",
+				"---Example 3 Start---",
+				"Investment \noption \nInvestment fees and \ncosts (p.a.) \n1 \nTransaction \ncosts (p.a.) \nMySuper/ \nBalanced \n0.38% (including 0.09% \nPerformance fee) \n0.18% \nManaged \nGrowth \n0.38% (including 0.11% \nPerformance fee) \n0.08% \n",
+				"---Example 3 End---",
+				"The column: \"Investment fees and costs (p.a.)\", \"including Performance fee\", meaning the value is the sum of \"Management costs\" and \"performance fee\", We should subtract the \"performance fee\" value, just output the \"Management costs\" value.",
+				"Both of management_fee and management_fee_and_costs are the values for \"Management costs\".",
+				"So, for fund: MySuper/Balanced, the value 0.38, including 0.09 Performance fee, so the Management costs is 0.38 - 0.09 = 0.29, performance_fee is 0.09.",
+				"For fund: Managed Growth, the value 0.38, including 0.11 Performance fee, so the Management costs is 0.38 - 0.11 = 0.27, performance_fee is 0.11.",
+				"So the output should be:",
+				"{\"data\": [{\"fund name\": \"MySuper/Balanced\", \"share name\": \"MySuper/Balanced\", \"management_fee_and_costs\": 0.29, \"management_fee\": 0.29, \"performance_fee\": 0.09}, {\"fund name\": \"Managed Growth\", \"share name\": \"Managed Growth\", \"management_fee_and_costs\": 0.27, \"management_fee\": 0.27, \"performance_fee\": 0.11}]}"
 			],
 			"administration_fees":[
 				"Administration fees and costs is share class level data.",
@@ -306,7 +356,7 @@
 			{
 				"title": "Don't fetch data with number range statement",
 				"contents":[
-					"If the value is with number range statement, e.g. \"up to\" or \"from to\" or \"between and\", please ignore the value.",
+					"If the value is with number range statement, e.g. \"up to\" or \"from to\" or \"between and\" or \"to\", please ignore the value.",
 					"Example 1:",
 					"-----Example Start-----",
 					"A-Class\nB-Class\nC-Class\n",

main.py

@@ -1521,7 +1521,7 @@ if __name__ == "__main__":
         # document_mapping_file = r"/data/aus_prospectus/basic_information/from_2024_documents/aus_100_document_prospectus_multi_fund.xlsx"
         # document_mapping_file = r"/data/aus_prospectus/basic_information/biz_rule/phase1_document_mapping.xlsx"
         document_mapping_file = r"/data/aus_prospectus/basic_information/17_documents/aus_prospectus_17_documents_mapping.xlsx"
-        # special_doc_id_list: list = ["420339794"]
+        # special_doc_id_list: list = ["411062815"]
         pdf_folder: str = r"/data/aus_prospectus/pdf/"
         output_pdf_text_folder: str = r"/data/aus_prospectus/output/pdf_text/"
         output_extract_data_child_folder: str = (