blade
/
dc-ml-emea-ar
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

complete configuration for AUS Prospectus

This commit is contained in:
Blade He 2025-01-07 16:25:13 -06:00
parent 201a809ffa
commit 0a867dcf07
11 changed files with 424 additions and 1232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

549
configuration/datapoint_keyword.json
View File

@ -1,526 +1,27 @@
{ {
"ISIN": { "total_annual_dollar_based_charges": {"english": ["total annual dollar based charges", "total annual dollar based charges ($)","total annual dollar"]},
"english": [ "management_fee_and_costs": {"english": ["management fees and cost"]},
"ISIN", "management_fee": {"english": ["management fee", "management fees","investment management fees","management fees and cost", "investment option management costs", "investment option management costs1"]},
"ISIN Code" "performance_fee": {"english": ["performance fee", "performance fees"]},
], "performance_fee_costs": {"english": ["performance fee costs", "performance fees costs"]},
"spanish": [ "buy_spread": {"english": ["buy-spread", "buy spread", "buy/sell spreads", "BUY-SELL SPREAD"]},
"ISIN", "sell_spread": {"english": ["sell-spread", "sell spread", "buy/sell spreads", "BUY-SELL SPREAD", "Buy:", "Sell:"]},
"ISIN Code" "establishment_fee": {"english": ["establishment fee", "establishment fees"]},
], "contribution_fee": {"english": ["contribution fee", "contribution fees"]},
"german": [ "withdrawal_fee": {"english": ["withdrawal fee", "withdrawal fees"]},
"ISIN", "switching_fee": {"english": ["switching fee", "switching fees"]},
"ISIN Code" "activity_fee": {"english": ["activity fee", "activity fees"]},
], "exit_fee": {"english": ["exit fee", "exit fees"]},
"dutch": [ "administration_fees": {"english": ["administration fee", "administration fees","admin fee"]},
"ISIN", "interposed_vehicle_performance_fee_cost": {"english": ["Performance fees charged by interposed vehicles","interposed vehicle performance fee cost", "interposed vehicle performance"]},
"ISIN Code" "additional_hurdle": {"english": ["additional hurdle","performance hardle"]},
], "benchmark_name": {"english": ["benchmark fund","benchmark name"]},
"french": [ "reference_rate": {"english": ["reference rate"]},
"ISIN", "crystallisation_frequency": {"english": ["crystallisation frequency"]},
"ISIN Code" "date_of_last_hwm_reset": {"english": ["date of last hwm reset"]},
], "date_of_last_performance_fee_restructure": {"english": ["date of last performance fee restructure"]},
"finnish": [ "high_water_mark_type": {"english": ["high-water mark type", "high water mark type"]},
"ISIN", "minimum_initial_investment": {"english": ["minimum initial investment","inital investment"]},
"ISIN Code" "recoverable_expenses": {"english": ["recoverable expenses","recoverable cost","expense recoveries"]},
], "indirect_costs": {"english": ["indirect cost","indirect fees","indirect costs"]}
"swedish": [
"ISIN",
"ISIN Code"
],
"danish": [
"ISIN",
"ISIN Code"
],
"norwegian": [
"ISIN",
"ISIN Code"
],
"lithuanian": [
"ISIN",
"ISIN Code"
],
"polish": [
"ISIN",
"ISIN Code"
],
"latvian": [
"ISIN",
"ISIN Code"
],
"india（keywords）": [
"ISIN",
"ISIN Code"
],
"estonian": [
"ISIN",
"ISIN Code"
],
"malay": [
"ISIN",
"ISIN Code"
],
"italian": [
"ISIN",
"ISIN Code"
],
"portuguese": [
"ISIN",
"ISIN Code"
]
},
"ter": {
"english": [
"Synthetic TER",
"Fund TER",
"TER",
"T.E.R",
"TER_REF",
"Total Expense Ratio",
"Total Expense Ratios",
"Expense ratio",
"Total Fund Charge",
"Gross Expense Ratio",
"Gross Expense Ratios",
"Capped Expense Ratio",
"all-in-fee",
"all-in fee",
"all in fee",
"Total Net Expense Ratio",
"Total Net Expense Ratios",
"Total Operating Expense",
"Expense Ratio",
"Expense Ratio -Direct",
"Expense Ratio -Regular",
"month End Expense ratio",
"expense ratio",
"Expenses Ratios",
"Weighted AverageExpense Ratio",
"Synthetic total Expense Ratio",
"Annualised TER including performance fees",
"TER (en %) (with performance)",
"Annualised TER % (with fee waiver)",
"expenses ratio",
"Total Expense as % of AAuM",
"Total Expenses as a % of AAuM",
"Recurring Expenses as a percentage to Average Net Assets",
"Total Expenses as % of AAuM",
"Income and Expenditure",
"Expenditure at Plan level as %",
"Total Expenses Inclusive of Management Fees of Direct Plan",
"Total Expenses Inclusive of Management Fees of Regular Plan"
],
"spanish": [
"Rácio da despesa total",
"Ratio Total de Gastos",
"Ratio de gastos totales",
"Porcentaje de gastos totales",
"Ratio de gastos totales"
],
"german": [
"Gesamtgebühren",
"Kostenpauschale",
"Gesamtkostenquote",
"Gesamtaufwandsquoten",
"Pauschalvergütung",
"GESAMTKOSTENANTEIL",
"kostenquote",
"Gesamt kostenquote",
"Betriebskostenquote des Fonds",
"Pauschalgebühr",
"Total Expense Ratio in Prozent",
"Annualisierte TER in % (Mit Gebührenverzicht)"
],
"dutch": [
"Totale-kostenpercentage",
"Totale Kostenratio",
"TKR",
"Totale kostenpercentage",
"Totaal kostenpercentage"
],
"french": [
"Le ratio de dépenses totales",
"Total des frais sur encours",
"TFE",
"Ratios des charges totales",
"Frais sur encours",
"RCT",
"Ratios des charges totales",
"Total des frais sur encours",
"TER",
"Ratio des dépenses totales",
"Ratio de dépenses totales",
"coefficienti di spesa totale",
"Total des commissions et frais imputés à"
],
"finnish": [
"palkkiot yhteensä",
"total expence ratio"
],
"swedish": [
"coefficienti di spesa totale",
"Totalkostnadsandel",
"TER"
],
"danish": [
"Administrationsomk",
"Omkostningsprocent",
"Totalkostnadsandel"
],
"norwegian": [
"Administrationsomk",
"Kostnadsratio",
"Kostnadsratioer for året",
"Omkostningsprocent"
],
"lithuanian": [
"Bendrųjų išlaidų koeficientas",
"Bendrasis metinis išlaidų rodikli"
],
"polish": [
"Współczynnik kosztów całkowitych",
"Bendrųjų išlaidų koeficientas",
"WKC"
],
"latvian": [
"Kopējo izdevumu koeficients",
"KIK",
"Kostnadsratio"
],
"india（keywords）": [
"Expenditure",
"expense ratio",
"ratio of",
"gross expense"
],
"estonian": [
"Kogukulude suhe",
"Kogukulude suhe aasta lõikes"
],
"malay": [
"NPP",
"Nisbah Perbelanjaan Pengurusan"
],
"italian": [
"Commissione forfetaria di gestione",
"Coefficienti di spesa totale",
"Coefficiente di spesa totale"
],
"portuguese": [
"Taxa Global de custos",
"Quocientes de Despesa Total",
"rácios de despesa total"
],
"hungarian": [
"Összes ráfordítás aránya"
]
},
"tor": {
"english": [
"TOR",
"Turnover Ratio",
"Turnover Rate",
"Portfolio Turnover",
"Portfolio turnover ratio",
"Portfolio turnover rate",
"PTR",
"Rotation",
"Annual Portfolio Turnover Ratio",
"Taux de rotation corrigé - Gecorrigeerde omloopsnelheid"
],
"india": [
"Aggregate Value of Purchase and Sale",
"The aggregate value of investments",
"The aggregate value of purchases",
"the aggregate of sales",
"Aggregate value of investments purchased and sold",
"The aggregate value of purchases and sales"
],
"spanish": [
"Rotación de la Cartera",
"Índice de rotación de la cartera",
"Ratio de rotación de la cartera"
],
"german": [
"Umschlagshaufigkeit",
"Portfolioumschlagshäufigkeit",
"Umschlagshäufigkeit",
"Portefeuilleumsatz",
"Portfolio Turnover Ratio",
"Umsatz",
"Portfolioumschlagsra",
"Umschlagkennziffer",
"Portfolioumschlag",
"Portfolioumschlagsrate"
],
"dutch": [
"Omloopsnelheid",
"Omloopfactor",
"Omloopsnelheid",
"Turnover van de portefeuille",
"Rotatie van de portefeuille",
"POF",
"Portefeuille omloop factor",
"Taux de rotation corrigé - Gecorrigeerde omloopsnelheid"
],
"french": [
"taux de rotation",
"Taux de rotation du portefeuille",
"Rotation de portefeuille",
"Ratio de rotation du portefeuille"
],
"finnish": [
"salkun kiertonopeus"
],
"swedish": [
"Omsättningshastighet"
],
"danish": [
"Omsætningshastighed"
],
"norwegian": [
"Omløpshastighet"
],
"india（keywords）": [
"Aggregate value"
],
"malay": [
"PGP",
"Pusing Ganti Portfolio"
],
"italian": [
"Tasso di movimentazione del portafoglio",
"Turnover del portafoglio",
"Indice di Rotazione del Portafoglio"
],
"portuguese": [
"Rotação média da carteira",
"Índice de Movimentação da Carteira de Investimento"
]
},
"ogc": {
"english": [
"Synthetic Ongoing Charges excluding",
"On-going Charge",
"Ongoing Charge",
"ongoing charges",
"On-going Fee",
"Ongoing fee",
"OGC",
"OGF",
"Operation Charge",
"On Going Charges",
"OC",
"Ongoing Charge Figure OCF",
"OCF Cap Rate",
"Ongoing Fund Charge",
"Operating Charge",
"Operating Charges",
"Operating, Administrative and Servicing Expenses"
],
"spanish": [
"Gastos Corrientes",
"Gastos Recurrentes"
],
"german": [
"Laufende Kosten",
"OCF",
"Ongoing Charge",
"Laufende Gebühren",
"laufende kosten in prozent",
"laufenden Kosten",
"Betriebskosten",
"Betriebsgebühren",
"custos correntes",
"Betriebliche Aufwendungen"
],
"dutch": [
"Lopende kosten",
"Lopende kosten factor",
"LKF",
"custos correntes",
"OCF",
"Lopende kosten factor"
],
"french": [
"Frais courants",
"Commission de frais opérationels"
],
"italian": [
"Spese Correnti",
"Durante il funzionamento Addebiti"
],
"portuguese": [
"Encargos Correntes",
"Custos correntes"
],
"swedish": [
"Årliga avgifter",
"pågående avgifter"
],
"danish": [
"Årlig avgift"
],
"norwegian": [
"Løpende gebyrer"
],
"malay": [
"Tρέχουσες επιβαρύνσεις"
]
},
"performance_fee": {
"english": [
"Performance Fee",
"Performance Fees",
"performance-based fee",
"performance-related fee",
"Performance- related Fee",
"perform- mance fees",
"per- formance fees",
"with performance",
"with performance fee",
"de Performance"
],
"spanish": [
"Comisión de Gestión sobre Resultados",
"Comisión sobre Resultados",
"Comisión de Rentabilidad",
"Comisiones de éxito",
"Comisión de Éxito",
"Comisión por resultados",
"comisión de rentabilidad",
"comisión de rendimiento"
],
"german": [
"Erfolgsabhängige Vergütung",
"Erfolgsabhängige Verwaltungsvergütung",
"Erfolgsbezogene Vergütung",
"Performancegebühren",
"Performancevergütung",
"Anlageerfolgsprämie",
"Anlageerfolgs-prämie",
"Anlageerfolgs- prämie",
"TER in % (inkl.",
"TER % (inkl.",
"TER in % (exkl.",
"TER % (exkl.",
"TER% (einschließlich",
"TER% (ohne",
"An die Wertentwicklung des Fonds gebundene Gebühren",
"Performancegebühr",
"Performance-gebühr",
"Erfolgshonorare",
"Erfolgsabhän-giger Vergütung",
"Erfolgshonorar",
"Performance-Fee",
"Erfolgsgebühr",
"perfolgsabhängige Verwaltungsvergütung",
"performanceabhängige Vergütung",
"Performance- gebühren"
],
"dutch": [
"Prestatievergoeding"
],
"french": [
"Les commissions de surperformance",
"Commission de performance",
"Commissions de surperformance",
"frais de performance"
],
"swedish": [
"Prestationsbaserad avgift",
"Performance-avgift"
],
"norwegian": [
"prestasjonsgebyr"
],
"italian": [
"Commissioni di performance",
"Commissioni legate al rendimento",
"Commissioni d’incentivo"
],
"portuguese": [
"Comissão de desempenho",
"Custos de performance",
"Comissão de Gestão Variável"
],
"estonian": [
" Edukustasud aasta lõikes"
],
"latvian": [
"Gada prēmijas par sasniegtajiem rezultātiem"
],
"Lithuanian": [
"Metinis mokestis už veiklos rezultatu"
]
},
"trading expense ratio": {
"english": [
"Trading expense ratio",
"Trading Expense Ratio10"
]
},
"mer": {
"english": [
"Management expense ratio",
"Management expense ratio after taxes",
"Expense ratio"
]
},
"MgtFee": {
"english": [
"Management Fee as % of AAuM",
"Management Fee including GST as % of AAuM",
"Management Fees",
"Management fee inclusive of service tax GST at annualised average rate",
"Management and Trusteeship Fees",
"Investment Management and Trusteeship fees",
"Investment management fees "
]
},
"max_management_fee": {
"english": [
"management fee",
"Periodic Charge",
"Advisory",
"max_management_fee"
]
},
"max_front_load": {
"english": [
"Sales charge",
"subscription fee",
"subscription charge",
"subscription commission",
"sales fee",
"entry fee",
"initial charge",
"preliminary charge",
"preliminary fee",
"Entry Charge",
"Initial Sales Charge",
"max_front_load"
]
},
"min_initial_purchase": {
"english": [
"Minimum Initial Subscription",
"Minimum Subscription",
"Minimum Subscription Amount",
"Minimum initial investment",
"min_initial_purchase"
]
},
"min_subsequent_purchase": {
"english": [
"Minimum Additional",
"Minimum Additional Subscription Amount",
"Minimum initial and subsequence subscription",
"Minimum Additional Subscription",
"Minimum Subsequent Investment",
"Minimum Subsequent Purchase",
"additional",
"min_subsequent_purchase"
]
}
} }

29
configuration/datapoint_level.json
View File

@ -1,6 +1,27 @@
{ {
"tor": "fund_level", "total_annual_dollar_based_charges": "share_level",
"ogc": "share_level", "management_fee_and_costs": "share_level",
"ter": "share_level", "management_fee": "share_level",
"performance_fee": "share_level" "performance_fee": "share_level",
"performance_fee_costs": "share_level",
"buy_spread": "share_level",
"sell_spread": "share_level",
"establishment_fee": "share_level",
"contribution_fee": "share_level",
"withdrawal_fee": "share_level",
"switching_fee": "share_level",
"activity_fee": "share_level",
"exit_fee": "share_level",
"administration_fees": "share_level",
"interposed_vehicle_performance_fee_cost": "share_level",
"additional_hurdle": "share_level",
"benchmark_name": "fund_level",
"reference_rate": "share_level",
"crystallisation_frequency": "share_level",
"date_of_last_hwm_reset": "share_level",
"date_of_last_performance_fee_restructure": "share_level",
"high_water_mark_type": "share_level",
"minimum_initial_investment": "share_level",
"recoverable_expenses": "share_level",
"indirect_costs": "share_level"
} }

29
configuration/datapoint_name.json
View File

@ -1,6 +1,27 @@
{ {
"tor": "TOR", "total_annual_dollar_based_charges": "total annual dollar based charges",
"ogc": "OGC", "management_fee_and_costs": "management fee and costs",
"ter": "TER", "management_fee": "management fee",
"performance_fee": "performance fees" "performance_fee": "performance fee",
"performance_fee_costs": "performance fee costs",
"buy_spread": "buy spread",
"sell_spread": "sell spread",
"establishment_fee": "establishment fee",
"contribution_fee": "contribution fee",
"withdrawal_fee": "withdrawal fee",
"switching_fee": "switching fee",
"activity_fee": "activity fee",
"exit_fee": "exit fee",
"administration_fees": "administration fee",
"interposed_vehicle_performance_fee_cost": "interposed vehicle performance fee cost",
"additional_hurdle": "additional hurdle",
"benchmark_name": "benchmark name",
"reference_rate": "reference rate",
"crystallisation_frequency": "crystallisation frequency",
"date_of_last_hwm_reset": "date of last hwm reset",
"date_of_last_performance_fee_restructure": "date of last performance fee restructure",
"high_water_mark_type": "high-water mark type",
"minimum_initial_investment": "minimum initial investment",
"recoverable_expenses": "recoverable expenses",
"indirect_costs": "indirect cost"
} }

417
configuration/datapoint_reported_name.json
View File

@ -1,392 +1,27 @@
{ {
"ter": { "total_annual_dollar_based_charges": {"english": ["total annual dollar based charges", "total annual dollar based charges ($)","total annual dollar"]},
"english": [ "management_fee_and_costs": {"english": ["management fees and cost", "management fees and costs", "management fee and cost"]},
"Synthetic TER", "management_fee": {"english": ["management fee", "management fees","investment management fees","management fees and cost", "investment option management costs", "investment option management costs1"]},
"Fund TER", "performance_fee": {"english": ["performance fee", "performance fees"]},
"TER", "performance_fee_costs": {"english": ["performance fee costs", "performance fees costs"]},
"TFE", "buy_spread": {"english": ["buy-spread", "buy spread", "buy/sell spreads", "BUY-SELL SPREAD"]},
"T.E.R", "sell_spread": {"english": ["sell-spread", "sell spread", "buy/sell spreads", "BUY-SELL SPREAD", "Buy:", "Sell:"]},
"TER_REF", "establishment_fee": {"english": ["establishment fee", "establishment fees"]},
"Total Expense Ratio", "contribution_fee": {"english": ["contribution fee", "contribution fees"]},
"Total Expense Ratios", "withdrawal_fee": {"english": ["withdrawal fee", "withdrawal fees"]},
"Total Fund Charge", "switching_fee": {"english": ["switching fee", "switching fees"]},
"Gross Expense Ratio", "activity_fee": {"english": ["activity fee", "activity fees"]},
"Gross Expense Ratios", "exit_fee": {"english": ["exit fee", "exit fees"]},
"Capped Expense Ratio", "administration_fees": {"english": ["administration fee", "administration fees","admin fee"]},
"all-in-fee", "interposed_vehicle_performance_fee_cost": {"english": ["Performance fees charged by interposed vehicles","interposed vehicle performance fee cost", "interposed vehicle performance"]},
"all in fee", "additional_hurdle": {"english": ["additional hurdle","performance hardle"]},
"Total Net Expense Ratio", "benchmark_name": {"english": ["benchmark fund","benchmark name"]},
"Total Net Expense Ratios", "reference_rate": {"english": ["reference rate"]},
"Total Operating Expense", "crystallisation_frequency": {"english": ["crystallisation frequency"]},
"Expense Ratio", "date_of_last_hwm_reset": {"english": ["date of last hwm reset"]},
"month End Expense ratio", "date_of_last_performance_fee_restructure": {"english": ["date of last performance fee restructure"]},
"Expenses Ratios", "high_water_mark_type": {"english": ["high-water mark type", "high water mark type"]},
"Weighted AverageExpense Ratio", "minimum_initial_investment": {"english": ["minimum initial investment","inital investment"]},
"Synthetic total Expense Ratio", "recoverable_expenses": {"english": ["recoverable expenses","recoverable cost","expense recoveries"]},
"Annualised TER including performance fees", "indirect_costs": {"english": ["indirect cost","indirect fees","indirect costs"]}
"TER (en %) (with performance)", }
"Annualised TER % (with fee waiver)"
],
"spanish": [
"Rácio da despesa total",
"Ratio Total de Gastos",
"Ratio de gastos totales",
"Porcentaje de gastos totales",
"Ratio de gastos totales"
],
"german": [
"Mit anteiliger Performance Fee in %",
"TER inkl. Performance-Fee in % **)",
"TER% (einschließlich Anlageerfolgsprämie)",
"TER % (inkl. Anlageerfolgsprämie)",
"Gesamtgebühren",
"Kostenpauschale",
"Gesamtkostenquote",
"Gesamtaufwandsquoten",
"Pauschalvergütung",
"GESAMTKOSTENANTEIL",
"kostenquote",
"Gesamt kostenquote",
"Betriebskostenquote des Fonds",
"Total Expense Ratio in Prozent",
"Annualisierte TER in % (Mit Gebührenverzicht)"
],
"dutch": [
"Totale-kostenpercentage",
"Totale Kostenratio",
"TKR",
"Totale kostenpercentage",
"Totaal kostenpercentage"
],
"french": [
"Le ratio de dépenses totales",
"Total des frais sur encours",
"Ratios des charges totales",
"Frais sur encours",
"RCT",
"Ratios des charges totales",
"Total des frais sur encours",
"Ratio des dépenses totales",
"Ratio de dépenses totales",
"coefficienti di spesa totale",
"Total des commissions et frais imputés à"
],
"finnish": [
"palkkiot yhteensä",
"total expence ratio"
],
"swedish": [
"coefficienti di spesa totale",
"Totalkostnadsandel",
"TER"
],
"danish": [
"Administrationsomk",
"Omkostningsprocent",
"Totalkostnadsandel"
],
"norwegian": [
"Administrationsomk",
"Kostnadsratio",
"Kostnadsratioer for året",
"Omkostningsprocent"
],
"lithuanian": [
"Bendrųjų išlaidų koeficientas",
"Bendrasis metinis išlaidų rodikli"
],
"polish": [
"Współczynnik kosztów całkowitych",
"Bendrųjų išlaidų koeficientas",
"WKC"
],
"latvian": [
"Kopējo izdevumu koeficients",
"KIK",
"Kostnadsratio"
],
"india（keywords）": [
"Expenditure",
"expense ratio",
"ratio of",
"gross expense"
],
"estonian": [
"Kogukulude suhe",
"Kogukulude suhe aasta lõikes"
],
"malay": [
"NPP",
"Nisbah Perbelanjaan Pengurusan"
],
"italian": [
"Commissione forfetaria di gestione",
"Coefficienti di spesa totale",
"Coefficiente di spesa totale"
],
"portuguese": [
"Taxa Global de custos",
"Quocientes de Despesa Total",
"rácios de despesa total"
],
"hungarian": [
"Összes ráfordítás aránya"
]
},
"tor": {
"english": [
"TOR",
"Turnover Ratio",
"Turnover Rate",
"Portfolio Turnover",
"Portfolio turnover ratio",
"Portfolio turnover rate",
"PTR",
"Rotation",
"Annual Portfolio Turnover Ratio",
"Taux de rotation corrigé - Gecorrigeerde omloopsnelheid"
],
"india": [
"Aggregate Value of Purchase and Sale",
"The aggregate value of investments",
"The aggregate value of purchases",
"the aggregate of sales",
"Aggregate value of investments purchased and sold",
"The aggregate value of purchases and sales"
],
"spanish": [
"Rotación de la Cartera",
"Índice de rotación de la cartera",
"Ratio de rotación de la cartera"
],
"german": [
"Umschlagshaufigkeit",
"Portfolioumschlagshäufigkeit",
"Umschlagshäufigkeit",
"Portefeuilleumsatz",
"Umsatz",
"Portfolioumschlagsra",
"Umschlagkennziffer",
"Portfolioumschlag",
"Portfolioumschlagsrate"
],
"dutch": [
"Omloopsnelheid",
"Omloopfactor",
"Omloopsnelheid",
"Turnover van de portefeuille",
"Rotatie van de portefeuille",
"POF",
"Portefeuille omloop factor",
"Taux de rotation - Omloopsnelheid"
],
"french": [
"taux de rotation",
"Taux de rotation du portefeuille",
"Rotation de portefeuille",
"Ratio de rotation du portefeuille"
],
"finnish": [
"salkun kiertonopeus"
],
"swedish": [
"Omsättningshastighet"
],
"danish": [
"Omsætningshastighed"
],
"norwegian": [
"Omløpshastighet"
],
"india（keywords）": [
"Aggregate value"
],
"malay": [
"PGP",
"Pusing Ganti Portfolio"
],
"italian": [
"Tasso di movimentazione del portafoglio",
"Turnover del portafoglio",
"Indice di Rotazione del Portafoglio"
],
"portuguese": [
"Rotação média da carteira",
"Índice de Movimentação da Carteira de Investimento"
]
},
"ogc": {
"english": [
"Synthetic Ongoing Charges excluding",
"On-going Charge",
"Ongoing Charge",
"ongoing charges",
"On-going Fee",
"Ongoing fee",
"OGC",
"OGF",
"OCF",
"Operation Charge",
"On Going Charges",
"OC",
"Ongoing Charge Figure OCF",
"OCF Cap Rate",
"Ongoing Fund Charge",
"Operating Charge",
"Operating Charges",
"Operating, Administrative and Servicing Expenses"
],
"spanish": [
"Gastos Corrientes",
"Gastos Recurrentes",
"Gastos corrientes en porcentaje",
"Gastos Corrientes 1)",
"Gastos Recurrentes 2)",
"Gastos corrientes en porcentaje 3)"
],
"german": [
"Ongoing Charges inkl.Performance-Fee in % **)",
"Laufende Kosten",
"Laufende Kosten in Prozent",
"Laufende Kosten 1)",
"Laufende Kosten in Prozent 2)",
"Laufende Gebühren",
"Betriebskosten",
"Betriebsgebühren",
"custos correntes",
"Betriebliche Aufwendungen"
],
"dutch": [
"Lopende kosten",
"Lopende kosten factor",
"Lopende kosten in procent",
"Lopende kosten factor 1)",
"Lopende kosten in procent 2)",
"LKF",
"custos correntes",
"OCF",
"Lopende kosten factor"
],
"french": [
"Frais courants",
"Frais courants exprimés en pourcentage",
"Frais courants 1)",
"Frais courants exprimés en pourcentage 2)",
"Commission de frais opérationels"
],
"italian": [
"Spese Correnti",
"Durante il funzionamento Addebiti",
"Spese Correnti 1)",
"Durante il funzionamento Addebiti 2)"
],
"portuguese": [
"Encargos Correntes",
"Custos correntes",
"Encargos Correntes 1)",
"Custos correntes 2)"
],
"swedish": [
"Årliga avgifter",
"pågående avgifter",
"Årliga avgifter 1)",
"pågående avgifter 2)"
],
"danish": [
"Årlig avgift",
"Årlig avgift 1)"
],
"norwegian": [
"Løpende gebyrer",
"Løpende gebyrer 1)"
],
"malay": [
"Tρέχουσες επιβαρύνσεις",
"Τρέχουσες επιβαρύνσεις 2)"
]
},
"performance_fee": {
"english": [
"Performance Fee",
"Performance Fees",
"performance-based fee",
"performance-related fee"
],
"spanish": [
"Comisión de Gestión sobre Resultados",
"Comisión sobre Resultados",
"Comisión de Gestión sobre Resultados 1)",
"Comisión sobre Resultados 2)",
"Comisión de Rentabilidad",
"Comisiones de éxito",
"Comisión de Éxito",
"Comisión por resultados",
"comisión de rentabilidad",
"comisión de rendimiento"
],
"german": [
"Erfolgsabhängige Vergütung",
"Erfolgsabhängige Verwaltungsvergütung",
"Erfolgsbezogene Vergütung",
"Performancegebühren",
"Erfolgsbezogene Vergütung 1)",
"Performancevergütung in Prozent",
"Performancevergütung in Prozent 2)",
"Anlageerfolgsprämie",
"An die Wertentwicklung des Fonds gebundene Gebühren",
"Performancegebühr",
"Erfolgshonorare",
"Erfolgshonorar",
"Erfolgsgebühr",
"perfolgsabhängige Verwaltungsvergütung",
"performanceabhängige Vergütung"
],
"dutch": [
"Prestatievergoeding",
"Prestatievergoeding 1)"
],
"french": [
"Les commissions de surperformance",
"Commission de performance",
"Commissions de surperformance",
"frais de performance",
"Commission de performance exprimée en pourcentage 2)"
],
"swedish": [
"Prestationsbaserad avgift",
"Performance-avgift",
"Prestationsbaserad avgift 1)",
"Performance-avgift 2)"
],
"norwegian": [
"prestasjonsgebyr",
"prestasjonsgebyr 1)"
],
"italian": [
"Commissioni di performance",
"Commissioni legate al rendimento",
"Commissioni di performance 1)",
"Commissioni legate al rendimento 2)",
"Commissioni d’incentivo"
],
"portuguese": [
"Comissão de desempenho",
"Custos de performance",
"Comissão de desempenho 1)",
"Custos de performance 2)",
"Comissão de Gestão Variável"
],
"estonian": [
"Edukustasud aasta lõikes",
"Edukustasud aasta lõikes 1)"
],
"latvian": [
"Gada prēmijas par sasniegtajiem rezultātiem",
"Gada prēmijas par sasniegtajiem rezultātiem 1)"
],
"Lithuanian": [
"Metinis mokestis už veiklos rezultatu",
"Metinis mokestis už veiklos rezultatu 2)"
]
}
}

13
configuration/document_dp_pages.json Normal file
View File

@ -0,0 +1,13 @@
{
"539790009": [39, 40, 45, 46, 47],
"542300403": [12],
"542301117": [17, 18],
"542306317": [4, 15, 16, 17, 18],
"547567013": [12, 13, 14, 15, 16, 17, 33, 34, 35, 39, 40, 42, 43, 44, 45],
"552505237": [16, 17, 18, 19, 25, 26, 27],
"552505278": [12, 13, 14, 15, 16, 17, 18, 19, 20, 27, 28, 29],
"554431052": [34, 35, 36, 41, 42, 43],
"554851189": [12, 13, 14],
"555377021": [21, 22, 23, 24, 25, 26],
"555654388": [35, 36]
}

29
configuration/domicile_datapoints.json
View File

@ -21,10 +21,31 @@
"performance_fee" "performance_fee"
], ],
"prospectus": [ "prospectus": [
"max_management_fee", "total_annual_dollar_based_charges",
"max_front_load", "management_fee_and_costs",
"min_initial_purchase", "management_fee",
"min_subsequent_purchase" "performance_fee",
"performance_fee_costs",
"buy_spread",
"sell_spread",
"establishment_fee",
"contribution_fee",
"withdrawal_fee",
"switching_fee",
"activity_fee",
"exit_fee",
"administration_fees",
"interposed_vehicle_performance_fee_cost",
"additional_hurdle",
"benchmark_name",
"reference_rate",
"crystallisation_frequency",
"date_of_last_hwm_reset",
"date_of_last_performance_fee_restructure",
"high_water_mark_type",
"minimum_initial_investment",
"recoverable_expenses",
"indirect_costs"
] ]
} }
} }

19
core/page_filter.py
View File

@ -25,7 +25,14 @@ class FilterPages:
self.datapoint_config, self.datapoint_exclude_config = ( self.datapoint_config, self.datapoint_exclude_config = (
self.get_datapoint_config() self.get_datapoint_config()
) )
self.get_document_dp_pages()
self.percentage_regex = r"\b\d{1,3}\.\d+\b|\b\d{1,3}(\,\d+)\b%" self.percentage_regex = r"\b\d{1,3}\.\d+\b|\b\d{1,3}(\,\d+)\b%"
def get_document_dp_pages(self) -> dict:
document_dp_pages_file = r"./configuration/document_dp_pages.json"
with open(document_dp_pages_file, "r", encoding="utf-8") as file:
self.document_dp_pages_config = json.load(file)
self.document_dp_pages = self.document_dp_pages_config.get(self.doc_id, [])
def get_pdf_page_text_dict(self) -> dict: def get_pdf_page_text_dict(self) -> dict:
pdf_util = PDFUtil(self.pdf_file) pdf_util = PDFUtil(self.pdf_file)
@ -142,9 +149,13 @@ class FilterPages:
result_details = [] result_details = []
for datapoint in self.datapoint_config.keys(): for datapoint in self.datapoint_config.keys():
result[datapoint] = [] result[datapoint] = []
for page_num, page_text in self.page_text_dict.items(): for page_index, page_text in self.page_text_dict.items():
if page_num < 2: if page_index < 2:
continue continue
page_num = page_index + 1
if self.document_dp_pages is not None and len(self.document_dp_pages) > 0 and page_num not in self.document_dp_pages:
continue
page_text = clean_text(page_text) page_text = clean_text(page_text)
text_split = [ text_split = [
sentence.strip() sentence.strip()
@ -179,11 +190,11 @@ class FilterPages:
is_valid = self.search_in_sentence_is_valid(search_text, text) is_valid = self.search_in_sentence_is_valid(search_text, text)
if not is_valid: if not is_valid:
continue continue
result[datapoint].append(page_num) result[datapoint].append(page_index)
detail = { detail = {
"doc_id": self.doc_id, "doc_id": self.doc_id,
"datapoint": datapoint, "datapoint": datapoint,
"page_num": page_num, "page_num": page_index,
"keyword": keyword, "keyword": keyword,
"text": search_text, "text": search_text,
} }

435
instructions/data_extraction_prompts_config.json
View File

@ -14,24 +14,10 @@
"3.1 The upon parsed text and table(s) contents as context.", "3.1 The upon parsed text and table(s) contents as context.",
"3.2 Please extract data from the context." "3.2 Please extract data from the context."
], ],
"reported_name": {
"tor": "The TOR reported name could be:\nTOR, Turnover Ratio, Portfolio Turnover, Portfolio turnover ratio, PTR, Taux de rotation corrigé - Gecorrigeerde omloopsnelheid, etc.",
"ogc": "The OGC reported name could be:\nOGC, OGF, OCF, Ongoing Charge, Operation Charge, Ongoing charges in per cent, Ongoing charges in percent, Ongoing charges as a percentage, On Going Charges, Operating Charge, Ongoing Fund Charge, OCF Cap Rate, Ongoing Charges Figure, Frais courants, Lopende kosten, Laufende Kosten, Årliga avgifter, Laufende Gebühren, Gastos Corrientes, Gastos Recurrentes, etc.",
"ter": "The TER reported name could be:\nTER, Total Expense Ratio, Total expense ratio as a percentage, Total Fund Charge, Gross Expense Ratio, All in fee, Total Net Expense Ratio, Weighted Average Expense Ratio, Synthetic total Expense Ratio, Annualised TER including performance fees, Capped Expense Ratio, TER (en %) (with performance), Net TER, Total Expense Ratio in Prozent, Annualisierte TER in % (Mit Gebührenverzicht), Annualised TER % (with fee waiver), kostenquote, Gesamt kostenquote, etc.",
"performance_fee": "The performance fees reported name could be:\nperformance fees, performance fees ratio, Performance, etc."
},
"multilingual_reported_name": {
"describe": "Please be careful to extract relevant data from multilingual Context.",
"regular_example_template": "{datapoint} Example {number}:\nLanguage: {language}\n---Context Start-----\n{fund_name}\n{share_name}\n{reported_name}\n{value}\n---Context End-----\nAnswer: {answer}",
"special_example_template_none": "{datapoint} Example {number}:\nLanguage: {language}\nIf value is belong to \"-, *, **, N/A, N/A%, N/A %, NONE\", ignore it\n---Context Start-----\n{fund_name}\n{share_name}\n{reported_name} 2)\n-\n---Context End-----\nAnswer: {answer}",
"value_examples": ["1,98", "3.25", "2.16", "1,73", "4,53"],
"fund_example": "Fund 1",
"share_example": "Share 1"
},
"data_business_features": { "data_business_features": {
"common": [ "common": [
"General rules:", "General rules:",
"- Most of cases, the data is in the table(s) of context.", "- The data is in the context, perhaps in table(s), semi-table(s) or paragraphs.",
"- Fund name: ", "- Fund name: ",
"a. The full fund name should be main fund name + sub-fund name, e,g, main fund name is Black Rock European, sub-fund name is Growth, the full fund name is: Black Rock European Growth.", "a. The full fund name should be main fund name + sub-fund name, e,g, main fund name is Black Rock European, sub-fund name is Growth, the full fund name is: Black Rock European Growth.",
"b. The sub-fund name may be as the first column or first row values in the table.", "b. The sub-fund name may be as the first column or first row values in the table.",
@ -51,51 +37,109 @@
"- Only extract the latest data from context:", "- Only extract the latest data from context:",
"If with multiple data values in same row, please extract the latest.", "If with multiple data values in same row, please extract the latest.",
"\n", "\n",
"d. Some table format, the fund name is in the end of row, please extract the fund name from the end of row.",
"---Example Start---",
"\nTotal\nTransaction Costs\nPerformance Fees\nManagement fees and costs\nIndirect Fee\nManagement fees\nMLC diversified investment\noption\n1.49% p.a.\n0.01% p.a.\n0.06% p.a.\n0.07% p.a.\n1.35% p.a.\nMLC Horizon 2\nIncome Portfolio\n",
"---Example End---",
"Correct fund name: MLC Horizon 2 Income Portfolio",
"\n",
"e. Fund and share relationship:",
"One fund could be with multiple share classes and relevant share class level data values.",
"If can't find specific share name, set share name as same as fund name.",
"---Example Start---",
"\nTotal\nTransaction Costs\nPerformance Fees\nManagement fees and costs\nIndirect Fee\nManagement fees\nMLC diversified investment\noption\n1.49% p.a.\n0.01% p.a.\n0.06% p.a.\n0.07% p.a.\n1.35% p.a.\nMLC Horizon 2\nIncome Portfolio\n",
"---Example End---",
"Correct fund name: MLC Horizon 2 Income Portfolio",
"Correct share name: MLC Horizon 2 Income Portfolio",
"- Reported names:", "- Reported names:",
"Only output the values which with significant reported names.", "Only output the values which with significant reported names.",
"Please exclude below reported names and relevant values: \"Management Fees\", \"Management\", \"Management Fees p.a.\", \"Taxe d Abonnement in % p.a.\".\nDON'T EXTRACT MANAGEMENT FEES!", "- Multiple data columns with same reported name but different post-fix:",
"One fund could be with multiple share classes and relevant share class level data values." "If there are multiple reported names with different post-fix text, here is the priority rule:",
"The pos-fix text is in the brackets: (gross), (net), pick up the values from (net).",
"---Example Start---",
"\n Investment option \nInvestment option \nmanagement \ncosts1 \n% p.a. \n(A)\nLifeplan \nadministration fee \n(gross)2 \n% p.a. \n(B)\nLifeplan \nadministration fee \n(net) \n% p.a. \n(C)\nTotal Management \nfees and costs \n(gross) \n% p.a. \n(A + B)\nTotal Management \nfees and costs \n(net) \n% p.a. \n(A + C)\nAllan Gray Australian Equity Fund \u2013 Class A\n0.77\n0.60\n0.42\n1.37\n1.19\n",
"---Example End---",
"The output should be:",
"{\"data\": [{\"fund name\": \"Allan Gray Australian Equity Fund\", \"share name\": \"Class A\", \"management_fee_and_costs\": 1.19, \"management_fee\": 0.77, \"administration_fees\": 0.42}]"
], ],
"investment_level": { "investment_level": {
"tor": "TOR is fund level data.", "total_annual_dollar_based_charges": "Total annual dollar based charges is share level data.",
"ogc": "OGC is share class level data", "management_fee_and_costs": "Management fee and costs is share level data.",
"ter": "TER is share class level data.", "management_fee": "Management fee is share level data.",
"performance_fee": "Performance fees is share class level data." "performance_fee_costs": "Performance fee costs is share class level data.",
"performance_fee": "Performance fees is share class level data.",
"buy_spread": "Buy spread is share class level data.",
"sell_spread": "Sell spread is share class level data.",
"establishment_fee": "Establishment fee is share class level data.",
"contribution_fee": "Contribution fee is share class level data.",
"withdrawal_fee": "Withdrawal fee is share class level data.",
"switching_fee": "Switching fee is share class level data.",
"activity_fee": "Activity fee is share class level data.",
"exit_fee": "Exit fee is share class level data.",
"administration_fees": "Administration fees is share class level data.",
"interposed_vehicle_performance_fee_cost": "Interposed vehicle performance fee cost is share class level data.",
"additional_hurdle": "Additional hurdle is share class level data.",
"benchmark_name": "Benchmark name is fund level data.",
"reference_rate": "Reference rate is share class level data.",
"crystallisation_frequency": "Crystallisation frequency is share class level data.",
"date_of_last_hwm_reset": "Date of last hwm reset is share class level data.",
"date_of_last_performance_fee_restructure": "Date of last performance fee restructure is share class level data.",
"high_water_mark_type": "High water mark type is share class level data.",
"minimum_initial_investment": "Minimum initial investment is share class level data.",
"recoverable_expenses": "Recoverable expenses is share class level data.",
"indirect_costs": "Indirect costs is share class level data."
}, },
"data_value_range": { "data_value_range": {
"tor": "TOR is belong to percentage number, the value could be more than 100, e.g. 126.33.\nTOR could be negative number, e.g. -7.99", "total_annual_dollar_based_charges": "Total annual dollar based charges is belong to decimal number, the value could be more than 100, e.g. 625.00",
"ogc": "OGC is belong to percentage number, the value should be less than 100.", "management_fee_and_costs": "Management fee and costs is belong to percentage number, the value should be less than 100.",
"ter": "TER is belong to percentage number, the value should be less than 100.", "management_fee": "Management fee is belong to percentage number, the value should be less than 100.",
"performance_fee": "Performance fees is belong to percentage number, the value should be less than 100.\nPerformance fees could be negative number, e.g. -0.56" "performance_fee_costs": "Performance fees costs is belong to percentage number, the value should be less than 100.",
"performance_fee": "Performance fees is belong to percentage number, the value should be less than 100.",
"buy_spread": "Buy spread is belong to percentage number, the value should be less than 100.",
"sell_spread": "Sell spread is belong to percentage number, the value should be less than 100.",
"establishment_fee": "Establishment fee is belong to percentage number, the value should be less than 100.",
"contribution_fee": "Contribution fee is belong to percentage number, the value should be less than 100.",
"withdrawal_fee": "Withdrawal fee is belong to percentage number, the value should be less than 100.",
"switching_fee": "Switching fee is belong to percentage number, the value should be less than 100.",
"activity_fee": "Activity fee is belong to percentage number, the value should be less than 100.",
"exit_fee": "Exit fee is belong to percentage number, the value should be less than 100.",
"administration_fees": "Administration fees is belong to percentage number, the value should be less than 100.",
"interposed_vehicle_performance_fee_cost": "Interposed vehicle performance fee cost is belong to percentage number, the value should be less than 100.",
"additional_hurdle": "Additional hurdle is belong to percentage number, the value should be less than 100.",
"benchmark_name": "Benchmark name is belong to index fund name, the value should be text.",
"reference_rate": "Reference rate is belong to percentage number, the value should be less than 100.",
"crystallisation_frequency": "Crystallisation frequency is belong to text, the value should be text.",
"date_of_last_hwm_reset": "Date of last hwm reset is belong to date, the value should be date format.",
"date_of_last_performance_fee_restructure": "Date of last performance fee restructure is belong to date, the value should be date format. e.g. 12 August 2022",
"high_water_mark_type": "High water mark type is belong to text, the value should be text.",
"minimum_initial_investment": "Minimum initial investment is belong to decimal number, the value could be more than 100, e.g. 625.00",
"recoverable_expenses": "Recoverable expenses is belong to percentage number, the value should be less than 100.",
"indirect_costs": "Indirect costs is belong to percentage number, the value should be less than 100."
}, },
"special_rule": { "special_rule": {
"tor": [ "management_fee_and_costs": [
"If there are multiple TOR reported names, here is the priority rules:", "If there are multiple Management fee and costs reported names, here is the priority rule:",
"- With \"Taux de rotation - Omloopsnelheid\" and \"Taux de rotation corrigé - Gecorrigeerde omloopsnelheid\", pick up the values from \"Taux de rotation - Omloopsnelheid\".", "- With \"Total Management fees and costs (gross)\" and \"Total Management fees and costs (net)\", pick up the values from \"Total Management fees and costs (net)\".",
"- With \"Omloopsnelheid\" and \"Gecorrigeerde omloopsnelheid\", pick up the values from \"Omloopsnelheid\"." "---Example Start---",
], "\n Investment option \nInvestment option \nmanagement \ncosts1 \n% p.a. \n(A)\nLifeplan \nadministration fee \n(gross)2 \n% p.a. \n(B)\nLifeplan \nadministration fee \n(net) \n% p.a. \n(C)\nTotal Management \nfees and costs \n(gross) \n% p.a. \n(A + B)\nTotal Management \nfees and costs \n(net) \n% p.a. \n(A + C)\nAllan Gray Australian Equity Fund \u2013 Class A\n0.77\n0.60\n0.42\n1.37\n1.19\n",
"ter": [ "---Example End---",
"If there are multiple TER value columns, here is the priority rules:", "The output should be:",
"- With \"TER with Performance Fee\" and \"Fund TER\", pick up the values from \"TER with Performance Fee\".", "{\"data\": [{\"fund name\": \"Allan Gray Australian Equity Fund\", \"share name\": \"Class A\", \"management_fee_and_costs\": 1.19, \"management_fee\": 0.77, \"administration_fees\": 0.42}]",
"- With \"TER (en %) (with performance)\" and \"TER(en %) (without performance)\", pick up the values from \"TER (en %) (with performance)\".", "\n",
"- With \"TER including Performance Fee\" and \"TER excluding Performance Fee\", pick up the values from \"TER including Performance Fee\".", "If there are multiple Management fee and costs sub-columns, here is the rule:",
"- With \"TER inkl. Performance-Fee in % **)\" and \"TER exkl. Performance-Fee in % **)\", pick up the values from \"TER inkl. Performance-Fee in % **)\".", "- With \"Management fees\" and \"Indirect fee\", sum the values from these two columns: \"Management fees\" + \"Indirect fee\".",
"- With \"TER inkl. Performance-Fee in % **)\" and \"TER inkl. Performance-Fee in % (inkl. Zielfonds)\", pick up the values from \"TER inkl. Performance-Fee in % **)\".", "---Example Start---",
"- With \"Mit anteiliger Performance Fee in %\" and \"Ohne anteilige Performance-Fee in %\", pick up the values from \"Mit anteiliger Performance Fee in %\".", "\nTotal\nTransaction Costs\nPerformance Fees\nManagement fees and costs\nIndirect Fee\nManagement fees\nMLC diversified investment\noption\n1.49% p.a.\n0.01% p.a.\n0.06% p.a.\n0.07% p.a.\n1.35% p.a.\nMLC Horizon 2\nIncome Portfolio\n",
"- With both of \"Synthetic TER\" and \"Fund TER\", if \"Synthetic TER\" with value(s), pick up the value(s) from \"Synthetic TER\", otherwise, pick up the value(s) from \"Fund TER\".", "---Example End---",
"- With both of \"Net TER\" and \"Capped Expense Ratio\", the priority is \"Net TER\", please exclude the column: \"Capped Expense Ratio\", only pick up the values from \"Net TER\".", "The output should be:",
"- With \"Gross TER\", \"Waiver\", \"Net TER\", \"Capped Expense Ratio\" as column titles, pick up the values from \"Net TER\".", "{\"data\": [{\"fund name\": \"MLC Horizon 2 Income Portfolio\", \"share name\": \"MLC Horizon 2 Income Portfolio\", \"management_fee_and_costs\": 1.42, \"management_fee\": 1.35, \"indirect_costs\": 0.07, \"performance_fee\": 0.06}]",
"- If exist Gross TER as column title, please ignore this title", "\n",
"Please ignore TER values which with the exception of performance fees or excluded performance fees." "- With \"Management fees\" and \"Administration fee\", sum the values from these two columns: \"Management fees\" + \"Administration fee\".",
], "---Example Start---",
"ogc": [ "\nTotal\nTransaction Costs\nPerformance Fees\nManagement fees and costs\nAdministration Fee\nManagement fees\nMLC diversified investment\noption\n1.62% p.a.\n0.02% p.a.\n0.03% p.a.\n0.09% p.a.\n1.58% p.a.\nMLC Horizon 4 Balanced\nPortfolio\n",
"If there are multiple OGC value columns, here is the priority rules:", "---Example End---",
"- With \"Ongoing Charges inkl. Performance-Fee in % **)\" and \"Ongoing Charges exkl. Performance-Fee in % **)\", pick up the values from \"Ongoing Charges inkl. Performance-Fee in % **)\".", "The output should be:",
"- With \"Ongoing Charges inkl. Performance-Fee in % **)\" and \"Ongoing Charges inkl. Performance-Fee in % (inkl. Zielfonds)\", pick up the values from \"Ongoing Charges inkl. Performance-Fee in % **)\"." "{\"data\": [{\"fund name\": \"MLC Horizon 4 Balanced Portfolio\", \"share name\": \"MLC Horizon 4 Balanced Portfolio\", \"management_fee_and_costs\": 1.67, \"management_fee\": 1.58, \"administration_fees\": 0.09, \"performance_fee\": 0.03}]"
],
"performance_fee": [
"The performance fees should not be the presence of the rates at which the performance fees are calculated.",
"The reported of performance fees should not be \"% based on the NAV at the end of the accounting period\""
] ]
} }
}, },
@ -110,34 +154,6 @@
"- Only extract data from the columns which column header is as the latest date.", "- Only extract data from the columns which column header is as the latest date.",
"-- commone case", "-- commone case",
"The latest date-time column usually is the first datapoint value column.", "The latest date-time column usually is the first datapoint value column.",
"-- special case",
"If with several value columns with same latest date, if one of these column titles with \"(c)\", please extract the data from this column.",
"---Example 1 Start---",
"Columns: \"For the year ended 31 Dec 23\", \"For the year ended 31 Dec 23\", \"For the year ended 31 Dec 23 (a)\", \"For the year ended 31 Dec 23 (b)\", \"For the year ended 31 Dec 23 (c)\", \"For the year ended 31 Dec 22\", \"For the year ended 31 Dec 21\", please extract the data from \"For the year ended 31 Dec 23 (c)\" column.",
"---Example 1 End---",
"---Example 2 Start---",
"Columns: \"For the period ended 31 Dec 23\", \"For the period ended 31 Dec 23\", \"For the period ended 31 Dec 23 (a)\", \"For the period ended 31 Dec 23 (b)\", \"For the period ended 31 Dec 23 (c)\", \"For the period ended 31 Dec 22\", \"For the period ended 31 Dec 21\", please extract the data from \"For the period ended 31 Dec 23 (c)\" column.",
"---Example 2 End---",
"More examples for extracting data from the latest date column:",
"-----Example Start-----",
"performance fees\\nhistorical performance fees\\nhistorical performance fees\\nFrom \\n1 July \\nFrom \\n19 July \\nFrom \\n1 January \\nFrom \\n27 April \\nFrom \\n19 July \\nFrom \\n1 January \\n2021\\nFrom \\n22 May \\n2021\\nFrom \\n16 July \\n2021\\nFrom \\n21 September \\n2021\\nto 30 June 2023\\nto 31 December 2022\\nto 31 December 2021\\nAsia Total Return Fund Class I5 (CHF Hedged) Acc\\n1.73%\\n \\n-1.32%\\n \\n \\n 2.04%\\n \\n \\n \\n",
"-----Example End-----",
"The output should be:",
"{\"data\": [{\"fund name\": \"Asia Total Return Fund\", \"share name\": \"Class I5 (CHF Hedged) Acc\", \"performance_fee\": 1.73}]}",
"The keywords are performance fees, the value 1.73 is the first number with the latest date-time.",
"Case 2:",
"Some table with messy text as header, please extract the data from the first 1 - 2 data value columns:",
"Example context:",
"-----Example Start-----",
"1RWHV WR WKH ILQDQFLDO VWDWHPHQWV Notes aux tats financiers\nLO Funds - 30/09/2023\n678 \n,6,1 &RGH \n6XE )XQGV \n6KDUH &ODVV \n)XQG 7(5 \n7(5 ZLWK \n3HUIRUPDQFH \n)HH \f \n6KDUH RI \n3HUIRUPDQFH \n)HH \n)XQG 7(5 \n7(5 ZLWK \n3HUIRUPDQFH \n)HH \f \n6KDUH RI \n3HUIRUPDQFH \n)HH \nCompartiments \nClasse \nTER du Fonds \nTER avec \nComm. de \nPerformance4) \nQuote part de la \nComm. de \nPerformance \nTER du Fonds \nTER avec \nComm. de \nPerformance4) \nQuote part de \nla Comm. de \nPerformance \n \f \n \f \n \f \n \f \n \f \n \f \n\b \n\b \n\b \n\b\n\b\n\b\nLU2376083999 \nTerreNeuve \nN A EUR SH X1 \n1.60 \n1.61 \n0.01 \n1.58 \n1.58 \n- \nLU1858044701 \nTerreNeuve \nN D GBP SH \n1.85 \n1.85 \n- \n1.84 \n1.86 \n- \n",
"-----Example End-----",
"Although the table is with messy text as header, but the latest date columns are the first 2 value columns, they are \"TER du Fonds\" and \"TER avec \nComm. de \nPerformance4\".",
"The TER value is from TER avec \nComm. de \nPerformance4, the performance fees value is from \"TER avec \nComm. de \nPerformance4\" - \"TER du Fonds\", e.g. 1.61 - 1.60 = 0.01, 1.85 - 1.85 = 0.",
"The output should be:",
"{\"data\": [{\"fund name\": \"TerreNeuve\", \"share name\": \"N A EUR SH X1\", \"ter\": 1.61, \"performance_fee\": 0.01}, {\"fund name\": \"TerreNeuve\", \"share name\": \"N D GBP SH\", \"ter\": 1.85, \"performance_fee\": 0}]}",
"Summary: \nIf there are several data value columns in the table, please extract the data from the latest date column(s).",
"If you are not sure which column is the latest date column, please extract the data from the first 1 - 2 data value columns.",
"Case 3:",
"If the value of column with latest date is N/A or -, please ignore.", "If the value of column with latest date is N/A or -, please ignore.",
"-----Example Start-----", "-----Example Start-----",
"I-class income shares\n\n31.10.22\n30.04.22\n30.04.21\n30.04.20\n\npence per share\npence per share\npence per share\npence per share\nOther information\nOperating charges**\nN/A\n—\n0.90%\n0.90%", "I-class income shares\n\n31.10.22\n30.04.22\n30.04.21\n30.04.20\n\npence per share\npence per share\npence per share\npence per share\nOther information\nOperating charges**\nN/A\n—\n0.90%\n0.90%",
@ -147,186 +163,35 @@
] ]
}, },
{ {
"title": "Don't fetch data with \"up to\" statement", "title": "Don't fetch data with number range statement",
"contents":[ "contents":[
"If the value is with \"up to\" statement, please ignore the value.", "If the value is with number range statement, e.g. \"up to\" or \"from to\" or \"between and\", please ignore the value.",
"Example 1:", "Example 1:",
"-----Example Start-----", "-----Example Start-----",
"A-Class\nB-Class\nC-Class\n", "A-Class\nB-Class\nC-Class\n",
"TER\nUp to 1.00%\nUp to 1.20%\nUp to 1.50%\n", "Management fee\nUp to 1.00%\nUp to 1.20%\nUp to 1.50%\n",
"-----Example End-----", "-----Example End-----",
"The output should be:", "The output should be:",
"{\"data\": []}", "{\"data\": []}",
"Example 2:", "Example 2:",
"-----Example Start-----", "-----Example Start-----",
"A-Aktien\nB-Aktien\nC-Aktien\n", "5 year average performance fees range estimated\nto be between 0.00% pa and 0.07% pa2,3.\n",
"TER\nbis zu 1,20 % p.a.\nbis zu 2,20 % p.a.\nbis zu 1,00 % p.a.\n", "-----Example End-----",
"The output should be:",
"{\"data\": []}",
"Example 3:",
"-----Example Start-----",
"Buy-sell spread ranges from 0.00% to 0.25%\ndepending on the investment option",
"-----Example End-----", "-----Example End-----",
"The output should be:", "The output should be:",
"{\"data\": []}" "{\"data\": []}"
] ]
} }
],
"ter": [
{
"title": "Combo TER value table:",
"contents": [
"Exist Feeder fund TER and Master fund TER.",
"The relevant table header is like this:",
"Feeder fund (share class)\\nMaster fund\\nTER\\nFeeder\\nTER Master\\nTotal",
"Please output separately as below:",
"- \"feeder fund share class\" and \"TER feeder\" values",
"- \"Master fund\" and \"TER Master\" values",
"Here is the example:",
"-----Example Start-----",
"Feeder fund (share class)\\nMaster fund\\nTER\\nFeeder\\nTER Master\\nTotal\\nGlobal Portfolio Solution DKK -\\nBalanced Class TI\\nDanske Invest SICAV Global Portfolio\\nSolution Balanced Class X\\n0.1475%\\n0.7025%\\n0.850%\\n",
"-----Example End-----",
"The output should be:",
"{\"data\": [{\"fund name\": \"Global Portfolio Solution DKK\", \"share name\": \"Balanced Class TI\", \"ter\": 0.1475}, {\"fund name\": \"Danske Invest SICAV Global Portfolio Solution DKK\", \"share name\": \"Balanced Class X\", \"ter\": 0.7025}]}"
]
},
{
"title": "TER reported name priority:",
"contents": [
"If exists both of Expense Ratio and Synthetic total Expense Ratio, please extract the value of Synthetic total Expense Ratio."
]
}
],
"performance_fee": [
{
"title": "Performance fees is part of TER:",
"contents": [
"Case 1:",
"If exist both of \"TER including performance fees\" and \"TER excluding performance fees\",",
"The TER should be \"TER including performance fees\".",
"The performance fees should be:",
"TER including performance fees - TER excluding performance fees.",
"Here is the example:",
"Example 1:",
"-----Example Start-----",
"GAMAX FUNDS FCP\\nClass\\nTER (excluding Performance Fees)\\nTER (including Performance Fees)\\nGAMAX FUNDS - ASIA PACIFIC\\nA\\n2.07%\\n2.07%\\n",
"-----Example End-----",
"The output should be:",
"{\"data\": [{\"fund name\": \"GAMAX FUNDS - ASIA PACIFIC\", \"share name\": \"A\", \"ter\": 2.07, \"performance_fee\": 0}]}",
"The performance fees value is TER (including Performance Fees) - TER (excluding Performance Fees) = 2.07 - 2.07 = 0",
"Example 2:",
"-----Example Start-----",
"D/S Strategie ausgewogen\\nErgänzende Angaben für Anleger in der Schweiz zum 31. Dezember 2020 (ungeprüft)\\nFonds\\nTER exkl. \\nPerformance-Fee in % **)\\nTER inkl. \\nPerformance-Fee in % **)\\nTER inkl. \\nPerformance-Fee in % (inkl. Zielfonds)\\n1,15\\n1,63\\n1,15\\n1,63\\nTER exkl.\\nPerformance-Fee in % (inkl. Zielfonds)",
"-----Example End-----",
"The output should be:",
"{\"data\": [{\"fund name\": \"D/S Strategie ausgewogen\", \"ter\": 1.15, \"performance_fee\": 0}]}",
"The performance fees value is TER inkl. Performance-Fee in % **) - TER exkl. Performance-Fee in % **) = 1,15 - 1,15 = 0",
"Example 3:",
"-----Example Start-----",
"TER % \n(inkl. \nAnlageerfolgsprämie)\nTER %\n(exkl. \nAnlageerfolgsprämie)\nPIANO 400 Fund\n0,58 %\n0,58 %\n",
"-----Example End-----",
"The output should be:",
"{\"data\": [{\"fund name\": \"PIANO 400 Fund\", \"ter\": 0.58, \"performance_fee\": 0}]}",
"The performance fees value is TER % (inkl. Anlageerfolgsprämie) - TER % (exkl. Anlageerfolgsprämie) = 0,58 - 0,58 = 0",
"Example 4:",
"-----Example Start-----",
"Fonds \nTER % \n(einschließlich \nAnlageerfolgs- \nprämie) \nTER % \n(ohne \nAnlageerfolgs-\nprämie) \ndb x-trackers EUR Liquid Corporate 12.5 UCITS ETF \n \n \nKlasse 1C \n0,35 % \n0,35 %",
"-----Example End-----",
"The output should be:",
"{\"data\": [{\"fund name\": \"db x-trackers EUR Liquid Corporate 12.5 UCITS ETF\", \"share name\": \"Klasse 1C\", \"ter\": 0.35, \"performance_fee\": 0}]}",
"The performance fees value is TER % (einschließlich Anlageerfolgsprämie) - TER % (ohne Anlageerfolgsprämie) = 0,35 - 0,35 = 0",
"or TER % (einschließlich Anlageerfolgs- \nprämie) - TER % (ohne Anlageerfolgs- \nprämie) = 0,35 - 0,35 = 0",
"Case 2:",
"If some table is with three value columns: \"TER including performance fees\", \"TER excluding performance fees\", \"Performance fees\", ",
"The Performance fees value in column: Performance fees, chould be \"-\", because of \"TER including performance fees\" - \"TER excluding performance fees\" = 0, ",
"But it's incorrect, according to this issue, please still extract performance fees from \"TER including performance fees\" - \"TER excluding performance fees\".",
"To make sure performance fees is with actual value.",
"Case 3:",
"If some table is with three value columns: \"TER including performance fees\", \"TER excluding performance fees\", \"SYNTHETIC TER\", ",
"The performace fee value is still \"TER including performance fees\" - \"TER excluding performance fees\", ",
"For this scenario, please ignore the \"SYNTHETIC TER\" column.",
"Here is the example:",
"-----Example Start-----",
"As at September 30, 2022, the annualised total expense ratios of \\nthe sub-fund Pictet - Corto Europe Long Short are as follows: \\nCLASS \\nANNUALISED TER INCLUDING \\nPERFORMANCE FEES \\nANNUALISED TER EXCLUDING \\nPERFORMANCE FEES \\nSYNTHETIC TER \\nP EUR \\n1.66% \\n1.66% \\n1.98%",
"-----Example End-----",
"The output should be:",
"{\"data\": [{\"fund name\": \"Pictet Corto Europe Long Short\", \"share name\": \"P EUR\", \"ter\": 1.98, \"performance_fee\": 0}]}",
"Attention: Please always output performance fee value including 0 after calculation as (TER including performance fees - TER excluding performance fees), although the value is 0, but it's with actual value."
]
},
{
"title": "Performance fees is part of OGC:",
"contents": [
"If exist both of \"Ongoing Charges including Performance Fee\" and \"Ongoing Charges excluding Performance Fee\",",
"The OGC should be \"Ongoing Charges including Performance Fee\".",
"The performance fees should be:",
"Ongoing Charges including Performance Fee - Ongoing Charges excluding Performance Fee.",
"Here is the example:",
"Example 1:",
"-----Example Start-----",
"GAMAX FUNDS FCP\\nClass\\Ongoing Charges (excluding Performance Fees)\\Ongoing Charges (including Performance Fees)\\nFund 1\\nShare A\\n1.50%\\n1.58%\\n",
"-----Example End-----",
"The output should be:",
"{\"data\": [{\"fund name\": \"Fund 1\", \"share name\": \"Share 1\", \"ogc\": 1.58, \"performance_fee\": 0.08}]}",
"The performance fees value is Ongoing Charges including Performance Fee - Ongoing Charges excluding Performance Fee = 1.58 - 1.50 = 0.08",
"Example 2:",
"-----Example Start-----",
"Fund1\\nOngoing Charges exkl. \\nPerformance-Fee in % **)\\nOngoing Charges exkl. \\nPerformance-Fee in % (inkl. Zielfonds)\\nOngoing Charges inkl. \\nPerformance-Fee in % **)\\nOngoing Charges inkl. \\nPerformance-Fee in % (inkl. Zielfonds)\\n1,15\\n1,35\\n1,20\\n1,35\\n",
"-----Example End-----",
"The output should be:",
"{\"data\": [{\"fund name\": \"Fund1\", \"ogc\": 1.20, \"performance_fee\": 0.05}]}",
"The performance fees value is Ongoing Charges inkl. Performance-Fee in % **) - Ongoing Charges exkl. Performance-Fee in % **) = 1.20 - 1.15 = 0.05"
]
}
],
"tor": [
{
"title": "TOR with TER and multiple years:",
"contents": [
"TOR and TER are in same table and with multiple years, please extract the TER and TOR value from the latest year column.",
"---Example 1 Start---",
"APPENDIX 1 – TOTAL EXPENSE RATIOS AND PORTFOLIO TURNOVER RATIOS\nTotal Expense Ratios are based on the trading 12 months preceding the dates listed below. \nTER \nPTR* \nFor the period/year ended \n2024\n2023\n2024\n2023\nYacktman \nClass A US$ \n1.70%\n1.71%\nTotal Sub-Fund \n(5.94)%\n(5.57)%\nDriehaus Emerging \nClass A US$ \n1.76%\n1.89%\nTotal Sub-Fund \n101.51%\n89.41%",
"---Example 1 End---",
"The output should be:",
"{\"data\": [{\"fund name\": \"Yacktman\", \"share name\": \"Class A US$\", \"ter\": 1.70}, {\"fund name\": \"Yacktman\", \"tor\": -5.94}, {\"fund name\": \"Driehaus Emerging\", \"share name\": \"Class A US$\", \"ter\": 1.76}, {\"fund name\": \"Driehaus Emerging\", \"tor\": 101.51}]}"
]
}
],
"extreme_complex": [
{
"title": "Complex Data Table Structure",
"regex": "([A-Z]{1,2}\\,\\s?){3,}",
"contents": [
"Complex Data Table Structure",
"Table structure: the first column is fund name, for each table title, there are a lot of share class names in it.",
"Please split these share class names and extract all of relevant data as fund name, share name, data point and value one by one from the table.",
"-----Example Start-----",
"Charges and expenses (continued) ",
"d) Operating, Administrative and Servicing Expenses / Operating Currency Hedged Share Class Fees (continued)",
"The following table shows the rates of Operating, Administrative and Servicing Expenses:",
"Class A, B, E, ",
"M,O ",
"EQUITY SUB-FUNDS ",
"a) Equity sub-funds ",
"Fund 1",
"0.35",
"Fund 2",
"0.26",
"-----Example End-----",
"The output should be:",
"{\"data\": [{\"fund name\": \"Fund 1\", \"share name\": \"A\", \"ogc\": 0.35},",
"{\"fund name\": \"Fund 1\", \"share name\": \"B\", \"ogc\": 0.35},",
"{\"fund name\": \"Fund 1\", \"share name\": \"E\", \"ogc\": 0.35},",
"{\"fund name\": \"Fund 1\", \"share name\": \"M\", \"ogc\": 0.35},",
"{\"fund name\": \"Fund 1\", \"share name\": \"O\", \"ogc\": 0.35}",
"{\"fund name\": \"Fund 2\", \"share name\": \"A\", \"ogc\": 0.26},",
"{\"fund name\": \"Fund 2\", \"share name\": \"B\", \"ogc\": 0.26},",
"{\"fund name\": \"Fund 2\", \"share name\": \"E\", \"ogc\": 0.26},",
"{\"fund name\": \"Fund 2\", \"share name\": \"M\", \"ogc\": 0.26},",
"{\"fund name\": \"Fund 2\", \"share name\": \"O\", \"ogc\": 0.26}]}"
]
}
] ]
}, },
"output_requirement": { "output_requirement": {
"common": [ "common": [
"If possible, please extract fund name, share name, TOR, TER, performance fees, OGC values as the output.", "If possible, please extract fund name, share name, data points values as the output.",
"If find share name, and exist relevant currency, please output share name + currency, e.g. share name is \"Class A\", currency is \"USD\", the output share name should be: \"Class A USD\".",
"If find fund name, and exist sub fund name, please output fund name + sub fund name, e.g. fund name is \"Black Rock European\", sub fund name is \"Growth\", the output fund name should be: \"Black Rock European Growth\".", "If find fund name, and exist sub fund name, please output fund name + sub fund name, e.g. fund name is \"Black Rock European\", sub fund name is \"Growth\", the output fund name should be: \"Black Rock European Growth\".",
"Only output the data point which with relevant value.", "Only output the data point which with relevant value.",
"Don't ignore the data point which with negative value, e.g. -0.12, -1.13", "Don't ignore the data point which with negative value, e.g. -0.12, -1.13",
@ -334,16 +199,16 @@
"Don't extract data which values are -, *, **, N/A, N/A%, N/A %, NONE, it means the value should be NULL, please skip them.", "Don't extract data which values are -, *, **, N/A, N/A%, N/A %, NONE, it means the value should be NULL, please skip them.",
"Please also output the data point reported name in context.", "Please also output the data point reported name in context.",
"Example:", "Example:",
"-----Example Start-----", "---Example Start---",
"Sub-Funds\nClass of shares\nCurrency\nTER\nPerformance\nfees\nSwiss Life Funds (LUX) Bond Emerging Markets Corporates\nAM - Shares CHF hedged - Capitalisation\nCHF\n0.23%\n-\nAM - Shares EUR hedged - Capitalisation\nEUR\n0.23%\n0.00%\n", "\n Investment option \nInvestment option \nmanagement \ncosts1 \n% p.a. \n(A)\nLifeplan \nadministration fee \n(gross)2 \n% p.a. \n(B)\nLifeplan \nadministration fee \n(net) \n% p.a. \n(C)\nTotal Management \nfees and costs \n(gross) \n% p.a. \n(A + B)\nTotal Management \nfees and costs \n(net) \n% p.a. \n(A + C)\nAllan Gray Australian Equity Fund \u2013 Class A\n0.77\n0.60\n0.42\n1.37\n1.19\nAlphinity Sustainable Share Fund\n0.95\n0.60\n0.42\n1.55\n1.37\nAntipodes Global Fund\n1.20\n0.60\n0.42\n1.80\n1.62\n",
"-----Example End-----", "---Example End---",
"Output:", "Output:",
"{\"data\": [{\"fund name\": \"Swiss Life Funds (LUX) Bond Emerging Markets Corporates\", \"share name\": \"AM - Shares CHF hedged - Capitalisation\", \"ter\": 0.23}, {\"fund name\": \"Swiss Life Funds (LUX) Bond Emerging Markets Corporates\", \"share name\": \"AM - Shares EUR hedged - Capitalisation\", \"ter\": 0.23, \"performance_fee\": 0}], \"dp_reported_name\": {\"ter\": \"TER\", \"performance_fee\": \"Performance\nfees\"}}", "{\"data\": [{\"fund name\": \"Allan Gray Australian Equity Fund\", \"share name\": \"Class A\", \"management_fee_and_costs\": 1.19, \"management_fee\": 0.77, \"administration_fees\": 0.42}, {\"fund name\": \"Alphinity Sustainable Share Fund\", \"share name\": \"Alphinity Sustainable Share Fund\", \"management_fee_and_costs\": 1.37, \"management_fee\": 0.95, \"administration_fees\": 0.42}, {\"fund name\": \"Antipodes Global Fund\", \"share name\": \"Antipodes Global Fund\", \"management_fee_and_costs\": 1.62, \"management_fee\": 1.20, \"administration_fees\": 0.42}]",
"Fund level data: (\"fund name\" and \"TOR\") and share level data: (\"fund name\", \"share name\", \"ter\", \"performance fees\", \"ogc\") should be output separately.", "Fund level data: (\"fund name\" and \"datapoint_name\") and share level data: (\"fund name\", \"share name\", \"datapoint_name\") should be output separately.",
"The output should be JSON format, the format is like below example(s):" "The output should be JSON format, the format is like below example(s):"
], ],
"fund_level": [ "fund_level": [
"[{\"fund name\": \"fund 1 - sub fund name 1\",\"tor\": 35.26}, {\"fund name\": \"fund 2 - sub fund name 2\",\"tor\": -28.26}, {\"fund name\": \"fund 3\",\"tor\": 115.52,}]" "[{\"fund name\": \"fund 1 - sub fund name 1\",\"benchmark_name\": \"S&P 500 Index Fund\"}, {\"fund name\": \"fund 2 - sub fund name 2\",\"benchmark_name\": \"FTSE All Share\"}]"
], ],
"share_level": { "share_level": {
"fund_name": [ "fund_name": [
@ -356,32 +221,62 @@
"share 2", "share 2",
"share 3" "share 3"
], ],
"ogc_value": [ "total_annual_dollar_based_charges_value": [125.00, 95.00, 26.00],
0.05, "management_fee_and_costs_value": [2.63, 1.58, 2.55],
1.08, "management_fee_value": [0.85, 1.10, 0.23],
0.17 "performance_fee_value": [0.03, 0.21, 0.08],
], "performance_fee_costs_value": [0.05, 0.25, 0.09],
"ter_value": [ "buy_spread_value": [0.10, 0.15, 0.12],
1.23, "sell_spread_value": [0.10, 0.10, 0.15],
2.56, "establishment_fee_value": [0.75, 1.20, 0.25],
1.16 "contribution_fee_value": [0.82, 0.10, 0.13],
], "withdrawal_fee_value": [0.75, 0.15, 1.23],
"performance_fee_value": [ "switching_fee_value": [0.25, 0.35, 1.53],
0, "activity_fee_value": [0.63, 0.25, 1.27],
-0.15, "exit_fee_value": [0.35, 0.18, 1.21],
0.11 "administration_fees_value": [0.10, 0.15, 0.08],
] "interposed_vehicle_performance_fee_cost_value": [0.02, 0.15, 0.03],
"additional_hurdle_value": [0.02, 0.15, 0.03],
"reference_rate_value": [0.02, 0.15, 0.03],
"crystallisation_frequency_value": ["Monthly", "Bi-Annually", "Annually"],
"date_of_last_hwm_reset_value": ["29 March 2023", "18 April 2024", "19 October 2021"],
"date_of_last_performance_fee_restructure_value": ["12 August 2022", "15 March 2024", "11 November 2023"],
"high_water_mark_type_value": ["Total Return", "Excess Return", "Both TR & ER"],
"minimum_initial_investment_value": [0, 5, 12],
"recoverable_expenses_value": [0.12, 0.05, 0.06],
"indirect_costs_value": [0.12, 0.16, 0.02]
}, },
"dp_reported_name" : { "dp_reported_name" : {
"tor": "TOR", "total_annual_dollar_based_charges": "Total annual dollar based charges",
"ogc": "OGC", "management_fee_and_costs": "Management fee and costs",
"ter": "TER", "management_fee": "Management fee",
"performance_fee": "Performance fees" "performance_fee": "Performance fee",
"performance_fee_costs": "Performance fee costs",
"buy_spread": "Buy spread",
"sell_spread": "Sell spread",
"establishment_fee": "Establishment fee",
"contribution_fee": "Contribution fee",
"withdrawal_fee": "Withdrawal fee",
"switching_fee": "Switching fee",
"activity_fee": "Activity fee",
"exit_fee": "Exit fee",
"administration_fees": "Administration fee",
"interposed_vehicle_performance_fee_cost": "Interposed vehicle performance fee cost",
"additional_hurdle": "Additional hurdle",
"benchmark_name": "Benchmark name",
"reference_rate": "Reference rate",
"crystallisation_frequency": "Crystallisation frequency",
"date_of_last_hwm_reset": "Date of last hwm reset",
"date_of_last_performance_fee_restructure": "Date of last performance fee restructure",
"high_water_mark_type": "High-water mark type",
"minimum_initial_investment": "Minimum initial investment",
"recoverable_expenses": "Recoverable expenses",
"indirect_costs": "Indirect cost"
} }
}, },
"end": [ "end": [
"Only output JSON data.", "Only output JSON data.",
"Don't output the value which not exist in context, especially for fund level datapoint: TOR.", "Don't output the value which not exist in context.",
"If can't find share class name in context, please output empty JSON data: {\"data\": []}" "If can't find fund name or share class name in context, please output empty JSON data: {\"data\": []}"
] ]
} }

96
main.py
View File

@ -132,10 +132,10 @@ class EMEA_AR_Parsing:
# Drilldown data to relevant PDF document # Drilldown data to relevant PDF document
annotation_list = [] annotation_list = []
try: # try:
annotation_list = self.drilldown_pdf_document(data_from_gpt) # annotation_list = self.drilldown_pdf_document(data_from_gpt)
except Exception as e: # except Exception as e:
logger.error(f"Error: {e}") # logger.error(f"Error: {e}")
return data_from_gpt, annotation_list return data_from_gpt, annotation_list
def drilldown_pdf_document(self, data_from_gpt: list) -> list: def drilldown_pdf_document(self, data_from_gpt: list) -> list:
@ -277,6 +277,7 @@ def mapping_data(
output_extract_data_folder: str, output_extract_data_folder: str,
output_mapping_folder: str, output_mapping_folder: str,
extract_way: str = "text", extract_way: str = "text",
drilldown_folder: str = r"/data/emea_ar/output/drilldown/",
re_run_extract_data: bool = False, re_run_extract_data: bool = False,
re_run_mapping_data: bool = False, re_run_mapping_data: bool = False,
) -> None: ) -> None:
@ -287,6 +288,7 @@ def mapping_data(
output_extract_data_folder=output_extract_data_folder, output_extract_data_folder=output_extract_data_folder,
output_mapping_data_folder=output_mapping_folder, output_mapping_data_folder=output_mapping_folder,
extract_way=extract_way, extract_way=extract_way,
drilldown_folder=drilldown_folder,
) )
doc_data_from_gpt, annotation_list = emea_ar_parsing.extract_data(re_run=re_run_extract_data) doc_data_from_gpt, annotation_list = emea_ar_parsing.extract_data(re_run=re_run_extract_data)
doc_mapping_data = emea_ar_parsing.mapping_data( doc_mapping_data = emea_ar_parsing.mapping_data(
@ -356,6 +358,7 @@ def batch_start_job(
output_extract_data_total_folder: str = r"/data/emea_ar/output/extract_data/total/", output_extract_data_total_folder: str = r"/data/emea_ar/output/extract_data/total/",
output_mapping_total_folder: str = r"/data/emea_ar/output/mapping_data/total/", output_mapping_total_folder: str = r"/data/emea_ar/output/mapping_data/total/",
extract_way: str = "text", extract_way: str = "text",
drilldown_folder: str = r"/data/emea_ar/output/drilldown/",
special_doc_id_list: list = None, special_doc_id_list: list = None,
re_run_extract_data: bool = False, re_run_extract_data: bool = False,
re_run_mapping_data: bool = False, re_run_mapping_data: bool = False,
@ -392,6 +395,7 @@ def batch_start_job(
output_extract_data_folder=output_extract_data_child_folder, output_extract_data_folder=output_extract_data_child_folder,
output_mapping_folder=output_mapping_child_folder, output_mapping_folder=output_mapping_child_folder,
extract_way=extract_way, extract_way=extract_way,
drilldown_folder=drilldown_folder,
re_run_extract_data=re_run_extract_data, re_run_extract_data=re_run_extract_data,
re_run_mapping_data=re_run_mapping_data, re_run_mapping_data=re_run_mapping_data,
) )
@ -869,18 +873,18 @@ def replace_rerun_data(new_data_file: str, original_data_file: str):
new_extract_data.to_excel(writer, index=False, sheet_name=extract_data_sheet) new_extract_data.to_excel(writer, index=False, sheet_name=extract_data_sheet)
def batch_run_documents(special_doc_id_list: list = None): def batch_run_documents(special_doc_id_list: list = None,
pdf_folder:str = r"/data/emea_ar/pdf/",
output_extract_data_child_folder:str = r"/data/emea_ar/output/extract_data/docs/",
output_extract_data_total_folder:str = r"/data/emea_ar/output/extract_data/total/",
output_mapping_child_folder:str = r"/data/emea_ar/output/mapping_data/docs/",
output_mapping_total_folder:str = r"/data/emea_ar/output/mapping_data/total/",
drilldown_folder: str = r"/data/emea_ar/output/drilldown/"):
sample_document_list_folder = r'./sample_documents/' sample_document_list_folder = r'./sample_documents/'
document_list_files = glob(sample_document_list_folder + "*.txt") document_list_files = glob(sample_document_list_folder + "*.txt")
pdf_folder = r"/data/emea_ar/pdf/"
page_filter_ground_truth_file = ( page_filter_ground_truth_file = (
r"/data/emea_ar/ground_truth/page_filter/datapoint_page_info_88_documents.xlsx" r"/data/emea_ar/ground_truth/page_filter/datapoint_page_info_88_documents.xlsx"
) )
output_extract_data_child_folder = r"/data/emea_ar/output/extract_data/docs/"
output_extract_data_total_folder = r"/data/emea_ar/output/extract_data/total/"
output_mapping_child_folder = r"/data/emea_ar/output/mapping_data/docs/"
output_mapping_total_folder = r"/data/emea_ar/output/mapping_data/total/"
re_run_extract_data = True re_run_extract_data = True
re_run_mapping_data = True re_run_mapping_data = True
force_save_total_data = False force_save_total_data = False
@ -890,8 +894,7 @@ def batch_run_documents(special_doc_id_list: list = None):
# special_doc_id_list = [] # special_doc_id_list = []
if special_doc_id_list is None or len(special_doc_id_list) == 0: if special_doc_id_list is None or len(special_doc_id_list) == 0:
force_save_total_data = True force_save_total_data = True
# file_base_name_candidates = ["sample_document_complex", "emea_case_from_word_complex"] file_base_name_candidates = []
file_base_name_candidates = ["sample_documents_12_11"]
for document_list_file in document_list_files: for document_list_file in document_list_files:
file_base_name = os.path.basename(document_list_file).replace(".txt", "") file_base_name = os.path.basename(document_list_file).replace(".txt", "")
if (file_base_name_candidates is not None and if (file_base_name_candidates is not None and
@ -909,6 +912,7 @@ def batch_run_documents(special_doc_id_list: list = None):
output_extract_data_total_folder, output_extract_data_total_folder,
output_mapping_total_folder, output_mapping_total_folder,
extract_way, extract_way,
drilldown_folder,
doc_id_list, doc_id_list,
re_run_extract_data, re_run_extract_data,
re_run_mapping_data, re_run_mapping_data,
@ -925,6 +929,7 @@ def batch_run_documents(special_doc_id_list: list = None):
output_extract_data_total_folder, output_extract_data_total_folder,
output_mapping_total_folder, output_mapping_total_folder,
extract_way, extract_way,
drilldown_folder,
special_doc_id_list, special_doc_id_list,
re_run_extract_data, re_run_extract_data,
re_run_mapping_data, re_run_mapping_data,
@ -933,18 +938,15 @@ def batch_run_documents(special_doc_id_list: list = None):
) )
def batch_initial_document(): def batch_initial_document(sample_document_list_folder: str = r'./sample_documents/',
sample_document_list_folder = r'./sample_documents/' document_list_file: str = "sample_document_complex.txt",
document_list_file = os.path.join(sample_document_list_folder, "japan_prospectus.txt") pdf_folder: str = r"/data/emea_ar/pdf/",
with open(document_list_file, "r", encoding="utf-8") as f: output_extract_data_child_folder: str = r"/data/emea_ar/output/extract_data/docs/",
output_mapping_child_folder: str = r"/data/emea_ar/output/mapping_data/docs/"):
document_list_file_path = os.path.join(sample_document_list_folder, document_list_file)
with open(document_list_file_path, "r", encoding="utf-8") as f:
doc_id_list = f.readlines() doc_id_list = f.readlines()
doc_id_list = [doc_id.strip() for doc_id in doc_id_list] doc_id_list = [doc_id.strip() for doc_id in doc_id_list]
pdf_folder = r"/data/illume/japan_prospectus/pdf/"
page_filter_ground_truth_file = (
r"/data/emea_ar/ground_truth/page_filter/datapoint_page_info_88_documents.xlsx"
)
output_extract_data_child_folder = r"/data/emea_ar/output/extract_data/docs/"
output_mapping_child_folder = r"/data/emea_ar/output/mapping_data/docs/"
for doc_id in tqdm(doc_id_list): for doc_id in tqdm(doc_id_list):
logger.info(f"Start to initial document: {doc_id}") logger.info(f"Start to initial document: {doc_id}")
emea_ar_parsing = EMEA_AR_Parsing(doc_id=doc_id, emea_ar_parsing = EMEA_AR_Parsing(doc_id=doc_id,
@ -1028,13 +1030,49 @@ def merge_output_data(data_file_path: str,
if __name__ == "__main__": if __name__ == "__main__":
data_file_path = r"/data/emea_ar/ground_truth/data_extraction/verify/20241211/sample_documents_12_11_mapping_data_info_44_documents_by_text_20241211185546.xlsx" # data_file_path = r"/data/emea_ar/ground_truth/data_extraction/verify/20241211/sample_documents_12_11_mapping_data_info_44_documents_by_text_20241211185546.xlsx"
document_mapping_file_path = r"/data/emea_ar/basic_information/sample_documents/sample_doc/sample_documents_12_11/doc_mapping_12_11.xlsx" # document_mapping_file_path = r"/data/emea_ar/basic_information/sample_documents/sample_doc/sample_documents_12_11/doc_mapping_12_11.xlsx"
output_data_file_path = r"/data/emea_ar/ground_truth/data_extraction/verify/20241211/sample_documents_12_11_merged_data_info.xlsx" # output_data_file_path = r"/data/emea_ar/ground_truth/data_extraction/verify/20241211/sample_documents_12_11_merged_data_info.xlsx"
# merge_output_data(data_file_path, document_mapping_file_path, output_data_file_path) # merge_output_data(data_file_path, document_mapping_file_path, output_data_file_path)
# batch_initial_document()
special_doc_id_list = ["553242411"] # sample_document_list_folder: str = r'./sample_documents/'
batch_run_documents(special_doc_id_list) # document_list_file: str = "aus_prospectus.txt"
# pdf_folder: str = r"/data/aus_prospectus/pdf/"
# output_extract_data_child_folder: str = r"/data/aus_prospectus/output/extract_data/docs/"
# output_mapping_child_folder: str = r"/data/aus_prospectus/output/mapping_data/docs/"
# batch_initial_document(sample_document_list_folder=sample_document_list_folder,
# document_list_file=document_list_file,
# pdf_folder=pdf_folder,
# output_extract_data_child_folder=output_extract_data_child_folder,
# output_mapping_child_folder=output_mapping_child_folder)
# special_doc_id_list = ["553242411"]
special_doc_id_list: list = ["539790009",
"542300403",
"542301117",
# "542306317",
"547567013",
"552505237",
"552505278",
"554431052",
"554851189",
"555377021",
"555654388"]
# special_doc_id_list: list = ["542300403"]
pdf_folder:str = r"/data/aus_prospectus/pdf/"
output_extract_data_child_folder:str = r"/data/aus_prospectus/output/extract_data/docs/"
output_extract_data_total_folder:str = r"/data/aus_prospectus/output/extract_data/total/"
output_mapping_child_folder:str = r"/data/aus_prospectus/output/mapping_data/docs/"
output_mapping_total_folder:str = r"/data/aus_prospectus/output/mapping_data/total/"
drilldown_folder = r"/data/aus_prospectus/output/drilldown/"
batch_run_documents(special_doc_id_list=special_doc_id_list,
pdf_folder=pdf_folder,
output_extract_data_child_folder=output_extract_data_child_folder,
output_extract_data_total_folder=output_extract_data_total_folder,
output_mapping_child_folder=output_mapping_child_folder,
output_mapping_total_folder=output_mapping_total_folder,
drilldown_folder=drilldown_folder)
# new_data_file = r"/data/emea_ar/output/mapping_data/total/mapping_data_info_15_documents_by_text_20241121154243.xlsx" # new_data_file = r"/data/emea_ar/output/mapping_data/total/mapping_data_info_15_documents_by_text_20241121154243.xlsx"
# original_data_file = r"/data/emea_ar/ground_truth/data_extraction/verify/mapping_data_info_30_documents_all_4_datapoints_20241106_verify_mapping.xlsx" # original_data_file = r"/data/emea_ar/ground_truth/data_extraction/verify/mapping_data_info_30_documents_all_4_datapoints_20241106_verify_mapping.xlsx"

29
prepare_data.py
View File

@ -1363,8 +1363,33 @@ def calc_typical_doc_metrics_v1():
) )
def merge_aus_document_prospectus_data():
"""
Merge AUS document and prospectus data.
"""
aus_document_file = r"/data/aus_prospectus/basic_information/document_mapping.xlsx"
aus_prospectus_file = r"/data/aus_prospectus/basic_information/aus_prospectus_data.xlsx"
aus_document_data = pd.read_excel(aus_document_file)
aus_prospectus_data = pd.read_excel(aus_prospectus_file)
aus_document_data["DocumentId"] = aus_document_data["DocumentId"].astype(str)
aus_document_prospectus_data = pd.merge(
aus_document_data,
aus_prospectus_data,
on=["FundClassId", "EffectiveDate"],
how="left",
)
aus_document_prospectus_file = r"/data/aus_prospectus/aus_document_prospectus.xlsx"
with pd.ExcelWriter(aus_document_prospectus_file) as writer:
aus_document_prospectus_data.to_excel(
writer, sheet_name="aus_document_prospectus", index=False
)
if __name__ == "__main__": if __name__ == "__main__":
# merge_aus_document_prospectus_data()
folder = r"/data/emea_ar/basic_information/English/sample_doc/emea_11_06_case/" folder = r"/data/emea_ar/basic_information/English/sample_doc/emea_11_06_case/"
file_name = "doc_ar_data_for_emea_11_06.xlsx" file_name = "doc_ar_data_for_emea_11_06.xlsx"
# get_document_with_all_4_data_points(folder, file_name, None) # get_document_with_all_4_data_points(folder, file_name, None)
@ -1400,8 +1425,8 @@ if __name__ == "__main__":
# doc_id_column="Document Id", # doc_id_column="Document Id",
# pdf_path=pdf_folder) # pdf_path=pdf_folder)
pdf_folder = r"/data/illume/japan_prospectus/pdf/" pdf_folder = r"/data/aus_prospectus/pdf/"
output_folder = r"/data/illume/japan_prospectus/pdf_txt/" output_folder = r"/data/aus_prospectus/pdf_txt/"
output_pdf_page_text(pdf_folder, output_folder) output_pdf_page_text(pdf_folder, output_folder)
# extract_pdf_table(pdf_folder, output_folder) # extract_pdf_table(pdf_folder, output_folder)

11
sample_documents/aus_prospectus.txt Normal file
View File

@ -0,0 +1,11 @@
547567013
539790009
542306317
542301117
542300403
552505278
554431052
555654388
554851189
555377021
552505237