595 lines
48 KiB
Plaintext
595 lines
48 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 29,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import openpyxl\n",
|
||
"from collections import defaultdict\n",
|
||
"import pandas as pd\n",
|
||
"import statistics\n",
|
||
"import os\n",
|
||
"import re\n",
|
||
"from utils.similarity import Similarity\n",
|
||
"\n",
|
||
"\n",
|
||
"imp_datapoints = [\"Management Fee and Costs\", \"Management Fee\", \"Performance fee and cost\", \"Interposed vehicle Performance fee and Costs\",\n",
|
||
" \"Administration Fee and costs\", \"Total Annual Dollar Based Charges\", \"Buy Spread\", \"Sell Spread\", \"Performance Fee\",\n",
|
||
" \"Minimum Initial Investment\", \"Benchmark\"]\n",
|
||
"\n",
|
||
"\n",
|
||
"imp_datapoints_mapping = {\n",
|
||
" \"Management Fee and Costs\": \"management_fee_and_costs\",\n",
|
||
" \"Management Fee\": \"management_fee\",\n",
|
||
" \"Performance fee and cost\": \"performance_fee_costs\",\n",
|
||
" \"Interposed vehicle Performance fee and Costs\": \"interposed_vehicle_performance_fee_cost\",\n",
|
||
" \"Administration Fee and costs\": \"administration_fees\",\n",
|
||
" \"Total Annual Dollar Based Charges\": \"total_annual_dollar_based_charges\",\n",
|
||
" \"Buy Spread\": \"buy_spread\",\n",
|
||
" \"Sell Spread\": \"sell_spread\",\n",
|
||
" \"Performance Fee\": \"PerformanceFeeCharged\",\n",
|
||
" \"Minimum Initial Investment\": \"minimum_initial_investment\",\n",
|
||
" \"Benchmark\": \"benchmark_name\"\n",
|
||
"}\n",
|
||
"\n",
|
||
"# imp_datapoints = [\"Management Fee and Costs\", \"Management Fee\", \"Performance fee and cost\",\n",
|
||
"# \"Administration Fee and costs\", \"Total Annual Dollar Based Charges\", \"Buy Spread\", \"Sell Spread\"]\n",
|
||
"\n",
|
||
"\n",
|
||
"# imp_datapoints_mapping = {\n",
|
||
"# \"Management Fee and Costs\": \"management_fee_and_costs\",\n",
|
||
"# \"Management Fee\": \"management_fee\",\n",
|
||
"# \"Performance fee and cost\": \"performance_fee_costs\",\n",
|
||
"# \"Administration Fee and costs\": \"administration_fees\",\n",
|
||
"# \"Total Annual Dollar Based Charges\": \"total_annual_dollar_based_charges\",\n",
|
||
"# \"Buy Spread\": \"buy_spread\",\n",
|
||
"# \"Sell Spread\": \"sell_spread\"\n",
|
||
"# }\n",
|
||
"\n",
|
||
"path_ground_truth = r\"/data/aus_prospectus/ground_truth/phase2_file/46_documents/46_documents_ground_truth_with_mapping.xlsx\"\n",
|
||
"# path_ground_truth = r\"/data/aus_prospectus/ground_truth/phase2_file/next_round/next_round_6_documents_ground_truth_with_mapping.xlsx\"\n",
|
||
"path_generated_results = r\"/data/aus_prospectus/output/mapping_data/total/mapping_data_info_46_documents_by_text_20250331220152.xlsx\"\n",
|
||
"# path_generated_results = r\"/data/aus_prospectus/output/mapping_data/total/mapping_data_info_46_documents_by_text_20250328035602.xlsx\"\n",
|
||
"# path_generated_results = r\"/data/aus_prospectus/output/mapping_data/total/mapping_data_info_6_documents_by_text_20250331180753.xlsx\"\n",
|
||
"provider_mapping_file_path = r\"/data/aus_prospectus/ground_truth/phase2_file/46_documents/TopProvidersBiz.xlsx\"\n",
|
||
"\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 30,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"\n",
|
||
"message_list = []\n",
|
||
"total_fn = []\n",
|
||
"def load_excel(filepath, header_row_index):\n",
|
||
" \"\"\"Load an Excel file and use the specified row as the header.\"\"\"\n",
|
||
" wb = openpyxl.load_workbook(filepath, data_only=True)\n",
|
||
" sheet = wb.active\n",
|
||
" headers = []\n",
|
||
" data = []\n",
|
||
"\n",
|
||
" for index, row in enumerate(sheet.iter_rows(values_only=True)):\n",
|
||
" if index == header_row_index:\n",
|
||
" headers = [cell if cell is not None else \"\" for cell in row]\n",
|
||
" elif index > header_row_index:\n",
|
||
" data.append([cell if cell is not None else \"\" for cell in row])\n",
|
||
"\n",
|
||
" return headers, data\n",
|
||
"\n",
|
||
"def index_data_by_key(data, header):\n",
|
||
" \"\"\"Index data by primary and secondary keys (doc_id and sec_name).\"\"\"\n",
|
||
" indexed_data = defaultdict(dict)\n",
|
||
" \n",
|
||
" for row in data:\n",
|
||
" row_data = {}\n",
|
||
" # Store the entire row, which will be useful for full row comparison\n",
|
||
" for i in range(len(row)):\n",
|
||
" if header[i] == \"doc_id\":\n",
|
||
" primary_key = int(row[i])\n",
|
||
" elif header[i] == \"sec_name\":\n",
|
||
" # share class should be the comparison level and key\n",
|
||
" secondary_key = str(row[i])\n",
|
||
" else:\n",
|
||
" row_data[header[i]] = convert_if_number(row[i])\n",
|
||
" if secondary_key is None or (isinstance(secondary_key, str) and len(secondary_key) == 0):\n",
|
||
" continue\n",
|
||
" indexed_data[primary_key][secondary_key] = row_data\n",
|
||
" return indexed_data\n",
|
||
"\n",
|
||
"def convert_if_number(value):\n",
|
||
" \"\"\"Attempt to convert value to a float or int, otherwise return as string.\"\"\"\n",
|
||
" try:\n",
|
||
" float_value = round(float(value), 2)\n",
|
||
" int_value = int(float_value)\n",
|
||
" return int_value if int_value == float_value else float_value\n",
|
||
" except (ValueError, TypeError):\n",
|
||
" return value\n",
|
||
"\n",
|
||
"def compare_values(value1, value2):\n",
|
||
" \"\"\"Convert values to numbers if possible and compare, otherwise compare as strings.\"\"\"\n",
|
||
" value1 = convert_if_number(value1)\n",
|
||
" value2 = convert_if_number(value2)\n",
|
||
" return value1 == value2\n",
|
||
"\n",
|
||
"def compare_data(ground_truth, generated_results, headers, intersection_list, document_list):\n",
|
||
" \"\"\"Compare data from two indexed sets, with the focus on matching generated results against ground truth.\"\"\"\n",
|
||
" results = {}\n",
|
||
" share_name_list = []\n",
|
||
" not_matched_share_name_list = []\n",
|
||
" share_matched, share_not_matched = 0, 0\n",
|
||
" # Initialize result dictionaries for each column except 'doc_id'\n",
|
||
" for keys in headers:\n",
|
||
" if keys != \"doc_id\":\n",
|
||
" results[keys] = {}\n",
|
||
" results[keys][\"TP\"] = 0\n",
|
||
" results[keys][\"TN\"] = 0\n",
|
||
" results[keys][\"FP\"] = 0\n",
|
||
" results[keys][\"FN\"] = 0\n",
|
||
" results[keys][\"SUPPORT\"] = 0\n",
|
||
" \n",
|
||
" # Iterate over the generated results instead of the ground truth\n",
|
||
" \n",
|
||
" total = 0\n",
|
||
" # print(document_list)\n",
|
||
" for doc_id, secs in ground_truth.items():\n",
|
||
" if document_list is not None and str(doc_id) not in document_list:\n",
|
||
" continue\n",
|
||
" if doc_id in generated_results:\n",
|
||
" for sec_name, truth_values in secs.items():\n",
|
||
" if sec_name in generated_results[doc_id]:\n",
|
||
" generated_values = generated_results[doc_id][sec_name]\n",
|
||
" # Compare all other columns\n",
|
||
" for i in intersection_list:\n",
|
||
" for keys in imp_datapoints:\n",
|
||
" if i == imp_datapoints_mapping[keys]:\n",
|
||
" truth = str(truth_values[i]).strip()\n",
|
||
" generated = str(generated_values[i]).strip()\n",
|
||
" total = total +1\n",
|
||
" if truth == \"\":\n",
|
||
" if truth == generated:\n",
|
||
" results[i][\"TN\"] = results[i][\"TN\"] + 1\n",
|
||
" else:\n",
|
||
" results[i][\"FP\"] = results[i][\"FP\"] + 1\n",
|
||
" # if \"Performance fee and cost\" in keys:\n",
|
||
" debug = 0\n",
|
||
" # print(keys, \" - \" , doc_id, \" truth is null and generated - \", generated_values[i], sec_name) \n",
|
||
" message = {\"data_point\": i, \"doc_id\": doc_id, \"sec_name\": sec_name, \n",
|
||
" \"truth\": truth, \"generated\": generated, \"error\": \"Truth is null and generated is not null\"}\n",
|
||
" message_list.append(message) \n",
|
||
" else:\n",
|
||
" if truth == generated:\n",
|
||
" results[i][\"TP\"] = results[i][\"TP\"] + 1\n",
|
||
" elif generated != \"\":\n",
|
||
" if i == \"benchmark_name\" and compare_text(truth, generated):\n",
|
||
" results[i][\"TP\"] = results[i][\"TP\"] + 1\n",
|
||
" else:\n",
|
||
" results[i][\"FP\"] = results[i][\"FP\"] + 1\n",
|
||
" # if \"Performance fee and cost\" in keys:\n",
|
||
" debug = 0\n",
|
||
" # print(keys, \" - \" , doc_id, \" truth - \", truth_values[i], \" and generated - \", generated_values[i], \" \", sec_name)\n",
|
||
" message = {\"data_point\": i, \"doc_id\": doc_id, \"sec_name\": sec_name, \n",
|
||
" \"truth\": truth, \"generated\": generated, \"error\": \"Truth is not equal with generated\"}\n",
|
||
" message_list.append(message)\n",
|
||
" else:\n",
|
||
" results[i][\"FN\"] = results[i][\"FN\"] + 1\n",
|
||
" # if \"Performance fee and cost\" in keys:\n",
|
||
" debug = 0\n",
|
||
" # print(keys, \" - \" , doc_id, \" generated is null and truth is - \", truth_values[i], sec_name)\n",
|
||
" message = {\"data_point\": i, \"doc_id\": doc_id, \"sec_name\": sec_name, \n",
|
||
" \"truth\": truth, \"generated\": generated, \"error\": \"Generated is null and truth is not null\"}\n",
|
||
" message_list.append(message)\n",
|
||
" results[i][\"SUPPORT\"] = results[i][\"SUPPORT\"] + 1\n",
|
||
" if sec_name not in share_name_list:\n",
|
||
" share_name_list.append(sec_name)\n",
|
||
" share_matched += 1\n",
|
||
" else:\n",
|
||
" if sec_name not in share_name_list:\n",
|
||
" share_name_list.append(sec_name)\n",
|
||
" if sec_name not in not_matched_share_name_list:\n",
|
||
" # If the share class is not found in the generated results, count it as not matched\n",
|
||
" # print(\"Share class not matched - \", sec_name, doc_id)\n",
|
||
" message = {\"data_point\": \"Share Class\", \"doc_id\": doc_id, \"sec_name\": sec_name, \n",
|
||
" \"truth\": \"\", \"generated\": \"\", \"error\": \"Share class not found in generated results\"}\n",
|
||
" message_list.append(message)\n",
|
||
" share_not_matched += 1\n",
|
||
" not_matched_share_name_list.append(sec_name)\n",
|
||
" else:\n",
|
||
" # If the entire document is not found, count all funds as not matched\n",
|
||
" message = {\"data_point\": \"Document\", \"doc_id\": doc_id, \"sec_name\": \"\",\n",
|
||
" \"truth\": \"\", \"generated\": \"\", \"error\": \"Document not found in generated results\"}\n",
|
||
" message_list.append(message)\n",
|
||
" \n",
|
||
" # share_not_matched += len(secs)\n",
|
||
" return results, message_list, share_matched, share_not_matched, not_matched_share_name_list\n",
|
||
"\n",
|
||
"def clean_text(text: str):\n",
|
||
" if text is None or len(text) == 0:\n",
|
||
" return text\n",
|
||
" text = re.sub(r\"\\W\", \" \", text)\n",
|
||
" text = re.sub(r\"\\s+\", \" \", text)\n",
|
||
" return text\n",
|
||
"\n",
|
||
"def compare_text(source_text, target_text):\n",
|
||
" source_text = clean_text(source_text)\n",
|
||
" target_text = clean_text(target_text)\n",
|
||
" if source_text == target_text or source_text in target_text or target_text in source_text:\n",
|
||
" return True\n",
|
||
" similarity = Similarity()\n",
|
||
" jacard_score = similarity.jaccard_similarity(source_text.lower().split(), target_text.lower().split())\n",
|
||
" if jacard_score > 0.8:\n",
|
||
" return True\n",
|
||
" \n",
|
||
" \n",
|
||
"def calculate_metrics(tp, tn, fp, fn):\n",
|
||
" \"\"\"Calculate precision, recall, accuracy, and F1-score.\"\"\"\n",
|
||
" precision = tp / (tp + fp) if (tp + fp) != 0 else 0\n",
|
||
" recall = tp / (tp + fn) if (tp + fn) != 0 else 0\n",
|
||
" accuracy = (tp + tn) / (tp + tn + fp + fn) if (tp + tn + fp + fn) != 0 else 0\n",
|
||
" f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) != 0 else 0\n",
|
||
" return precision, recall, accuracy, f1_score\n",
|
||
"\n",
|
||
"def print_metrics_table(data):\n",
|
||
" # Print table headers\n",
|
||
" print(\"{:<50}\\t{:<10}\\t{:<10}\\t{:<10}\\t{:<10}\\t{:<10}\\t{:<10}\\t{:<10}\\t{:<10}\\t{:<10}\".format(\"Metric\", \"F1-Score\", \"Precision\", \"Recall\", \"Accuracy\", \"SUPPORT\", \"TP\", \"TN\", \"FP\", \"FN\"))\n",
|
||
" total_precision, total_recall, total_accuracy, total_f1_score, total_support= [],[],[],[],[]\n",
|
||
" \n",
|
||
" total_tp = []\n",
|
||
" total_tn = []\n",
|
||
" total_fp = []\n",
|
||
" #total_fn = []\n",
|
||
" # Calculate and print metrics for each item\n",
|
||
" metrics_list = []\n",
|
||
" for keys in imp_datapoints:\n",
|
||
" try:\n",
|
||
" key = imp_datapoints_mapping[keys]\n",
|
||
" values = data[key]\n",
|
||
" if values[\"SUPPORT\"] == 0:\n",
|
||
" continue\n",
|
||
" tp, tn, fp, fn = values['TP'], values['TN'], values['FP'], values['FN']\n",
|
||
" precision, recall, accuracy, f1_score = calculate_metrics(tp, tn, fp, fn)\n",
|
||
" metrics = {\"Datapoint\": key, \"F1-Score\": f1_score, \"Precision\": precision, \"Recall\": recall, \"Accuracy\": accuracy, \"SUPPORT\": values[\"SUPPORT\"], \"TP\": tp, \"TN\": tn, \"FP\": fp, \"FN\": fn}\n",
|
||
" metrics_list.append(metrics)\n",
|
||
" total_precision.append(precision)\n",
|
||
" total_recall.append(recall)\n",
|
||
" total_accuracy.append(accuracy)\n",
|
||
" total_f1_score.append(f1_score)\n",
|
||
" total_support.append(values[\"SUPPORT\"])\n",
|
||
" total_tp.append(tp)\n",
|
||
" total_tn.append(tn)\n",
|
||
" total_fp.append(fp)\n",
|
||
" total_fn.append(fn)\n",
|
||
"\n",
|
||
" if values[\"SUPPORT\"] > 0 and key > \"\":\n",
|
||
" print(\"{:<50}\\t{:<10.4f}\\t{:<10.4f}\\t{:<10.4f}\\t{:<10.4f}\\t{:<10.0f}\\t{:<10.0f}\\t{:<10.0f}\\t{:<10.0f}\\t{:<10.0f}\".format(key, f1_score, precision, recall, accuracy, values[\"SUPPORT\"], tp, tn, fp, fn))\n",
|
||
" except:\n",
|
||
" pass\n",
|
||
" total_mean_precision = statistics.mean(total_precision)\n",
|
||
" total_mean_recall = statistics.mean(total_recall)\n",
|
||
" total_mean_accuracy = statistics.mean(total_accuracy)\n",
|
||
" total_mean_f1_score = statistics.mean(total_f1_score)\n",
|
||
" total_sum_support = sum(total_support)\n",
|
||
" total_sum_tp = sum(total_tp)\n",
|
||
" total_sum_tn = sum(total_tn)\n",
|
||
" total_sum_fp = sum(total_fp)\n",
|
||
" total_sum_fn = sum(total_fn)\n",
|
||
" total_metrics = {\"Datapoint\": \"TOTAL\", \"F1-Score\": total_mean_f1_score, \"Precision\": total_mean_precision, \"Recall\": total_mean_recall, \"Accuracy\": total_mean_accuracy, \"SUPPORT\": total_sum_support, \"TP\": total_sum_tp, \"TN\": total_sum_tn, \"FP\": total_sum_fp, \"FN\": total_sum_fn}\n",
|
||
" metrics_list.append(total_metrics)\n",
|
||
" print(\"{:<50}\\t{:<10.4f}\\t{:<10.4f}\\t{:<10.4f}\\t{:<10.4f}\\t{:<10.0f}\\t{:<10.0f}\\t{:<10.0f}\\t{:<10.0f}\\t{:<10.0f}\".format(\"TOTAL\", total_mean_f1_score, total_mean_precision, total_mean_recall, total_mean_accuracy, total_sum_support, total_sum_tp, total_sum_tn, total_sum_fp, total_sum_fn))\n",
|
||
" return metrics_list\n",
|
||
" \n",
|
||
"def create_metrics_df(data):\n",
|
||
" # Define a list to hold data for DataFrame\n",
|
||
" rows = []\n",
|
||
" \n",
|
||
" # Iterate through each metric item\n",
|
||
" for key in imp_datapoints:\n",
|
||
" try:\n",
|
||
" mapped_key = imp_datapoints_mapping[key]\n",
|
||
" values = data[mapped_key]\n",
|
||
" tp, tn, fp, fn = values['TP'], values['TN'], values['FP'], values['FN']\n",
|
||
" precision, recall, accuracy, f1_score = calculate_metrics(tp, tn, fp, fn)\n",
|
||
" \n",
|
||
" # Only add rows where SUPPORT > 0\n",
|
||
" if values[\"SUPPORT\"] > 0:\n",
|
||
" row = {\n",
|
||
" \"Metric\": key,\n",
|
||
" \"Precision\": precision,\n",
|
||
" \"Recall\": recall,\n",
|
||
" \"Accuracy\": accuracy,\n",
|
||
" \"F1-Score\": f1_score,\n",
|
||
" \"SUPPORT\": values[\"SUPPORT\"]\n",
|
||
" }\n",
|
||
" rows.append(row)\n",
|
||
" except KeyError as e:\n",
|
||
" continue\n",
|
||
"\n",
|
||
" # Create a DataFrame from the list of rows\n",
|
||
" df_metrics = pd.DataFrame(rows)\n",
|
||
" df_metrics.reset_index(inplace=True)\n",
|
||
" df_metrics.drop(columns=[\"index\"], inplace=True)\n",
|
||
" print(df_metrics)\n",
|
||
" return df_metrics\n",
|
||
"\n",
|
||
"\n",
|
||
"\n",
|
||
"def get_provider_mapping(file_path):\n",
|
||
" df = pd.read_excel(file_path)\n",
|
||
" df = (df.groupby([\"Docid\", \"ProviderName\"]).first())\n",
|
||
" df.reset_index(inplace = True)\n",
|
||
" return df[[\"Docid\", \"ProviderName\"]]\n",
|
||
"\n",
|
||
"\n",
|
||
"def get_provider_names(generated_results_indexed, df_provider_mapping):\n",
|
||
" providers_dict = {}\n",
|
||
" for doc_id in generated_results_indexed:\n",
|
||
" try:\n",
|
||
" provider_name = (df_provider_mapping[df_provider_mapping[\"Docid\"] == doc_id][\"ProviderName\"].values)[0]\n",
|
||
" if provider_name in providers_dict:\n",
|
||
" providers_dict[provider_name].append(doc_id)\n",
|
||
" else:\n",
|
||
" providers_dict[provider_name] = []\n",
|
||
" providers_dict[provider_name].append(doc_id)\n",
|
||
"\n",
|
||
" except:\n",
|
||
" pass\n",
|
||
" return providers_dict\n",
|
||
"\n",
|
||
"def get_specified_doc_data(results, doc_list):\n",
|
||
" provider_res = {}\n",
|
||
" for doc_id in doc_list:\n",
|
||
" if doc_id in results:\n",
|
||
" provider_res[doc_id] = results[doc_id]\n",
|
||
" return provider_res\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 31,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"\n",
|
||
"\n",
|
||
"\n",
|
||
"All Providers Results: \n",
|
||
"Document List File - None\n",
|
||
"Metric \tF1-Score \tPrecision \tRecall \tAccuracy \tSUPPORT \tTP \tTN \tFP \tFN \n",
|
||
"management_fee_and_costs \t0.9369 \t0.8988 \t0.9785 \t0.8814 \t413 \t364 \t0 \t41 \t8 \n",
|
||
"management_fee \t0.9478 \t0.9185 \t0.9789 \t0.9007 \t413 \t372 \t0 \t33 \t8 \n",
|
||
"performance_fee_costs \t0.9160 \t0.9231 \t0.9091 \t0.8935 \t273 \t240 \t129 \t20 \t24 \n",
|
||
"interposed_vehicle_performance_fee_cost \t0.9114 \t0.8372 \t1.0000 \t0.9661 \t73 \t72 \t327 \t14 \t0 \n",
|
||
"administration_fees \t1.0000 \t1.0000 \t1.0000 \t1.0000 \t52 \t52 \t361 \t0 \t0 \n",
|
||
"total_annual_dollar_based_charges \t1.0000 \t1.0000 \t1.0000 \t1.0000 \t52 \t52 \t361 \t0 \t0 \n",
|
||
"buy_spread \t0.9290 \t0.8920 \t0.9691 \t0.8838 \t359 \t314 \t51 \t38 \t10 \n",
|
||
"sell_spread \t0.9258 \t0.8864 \t0.9689 \t0.8789 \t359 \t312 \t51 \t40 \t10 \n",
|
||
"minimum_initial_investment \t0.9733 \t0.9799 \t0.9669 \t0.9613 \t302 \t292 \t105 \t6 \t10 \n",
|
||
"benchmark_name \t0.9109 \t0.8790 \t0.9452 \t0.9346 \t155 \t138 \t248 \t19 \t8 \n",
|
||
"TOTAL \t0.9451 \t0.9215 \t0.9717 \t0.9300 \t2451 \t2208 \t1633 \t211 \t78 \n",
|
||
"Total Shares Matched - 365\n",
|
||
"Total Shares Not Matched - 142\n",
|
||
"Percentage of Shares Matched - 71.99211045364892\n",
|
||
"Not Matched Shares Name List - ['SPDR® S&P World ex Australia Carbon Control Fund', 'Mercer Multi-manager Growth Fund – Retail Units', 'Mercer Multi-manager High Growth Fund – Retail Units', 'ANZ OA IP-OP Diversified Credit EF', 'ANZ OA IP-OP Diversified Credit NE', 'OnePath ANZ OA IP-T. Rowe Price Dyna Gl Bond EF', 'OnePath ANZ OA IP-T. Rowe Price Dyna Gl Bond NE', 'OnePath OA IP- Pendal Monthly Income Plus-EF/Sel', 'OnePath OA IP-ANZ Cash Advantage-EF/Sel', 'OnePath OA IP-ANZ Cash Advantage-NEF', 'OnePath OA IP-Ausbil Australian Emerging Leaders Trust-EF/Sel', 'OnePath OA IP-Bentham Global Income Trust-EF/Sel', 'OnePath OA IP-Bentham Global Income Trust-NEF', 'OnePath OA IP-Fidelity Australian Equities-EF/Sel', 'OnePath OA IP-Fidelity Australian Equities-NEF', 'OnePath OA IP-Investors Mutual Australian Share Trust- EF/Sel', 'OnePath OA IP-Investors Mutual Australian Share Trust- NEF', 'OnePath OA IP-Kapstream Absolute Return Income Trust-EF/Sel', 'OnePath OA IP-Kapstream Absolute Return Income Trust-NEF', 'OnePath OA IP-Merlon Australian Share Income-EF/Sel', 'OnePath OA IP-OnePath Active Growth Trust-NEF', 'OnePath OA IP-OnePath High Growth Trust-EF/Sel', 'OnePath OA IP-OnePath High Growth Trust-NEF', 'OnePath OA IP-OnePath Managed Growth Trust-EF/Sel', 'OnePath OA IP-OnePath Managed Growth Trust-NEF', 'OnePath OA IP-OptiMix Australian Fixed Interest Trust-EF/Sel', 'OnePath OA IP-OptiMix Australian Fixed Interest Trust-NEF', 'OnePath OA IP-OptiMix Australian Share Trust-EF/Sel', 'OnePath OA IP-OptiMix Australian Share Trust-NEF', 'OnePath OA IP-OptiMix Global Emerging Markets Share-EF/Sel', 'OnePath OA IP-OptiMix Global Emerging Markets Share-NEF', 'OnePath OA IP-OptiMIx Global Share Trust-EF/Sel', 'OnePath OA IP-OptiMIx Global Share Trust-NEF', 'OnePath OA IP-OptiMix High Growth Trust-EF/Sel', 'OnePath OA IP-OptiMix High Growth Trust-NEF', 'OnePath OA IP-OptiMix Property Securities Trust-EF/Sel', 'OnePath OA IP-OptiMix Property Securities Trust-NEF', 'OnePath OA IP-Perpetual Balanced Growth Trust-EF/Sel', 'OnePath OA IP-Perpetual Balanced Growth Trust-NEF', 'OnePath OA IP-Perpetual Conservative Growth Trust-EF/Sel', 'OnePath OA IP-Perpetual Conservative Growth Trust-NEF', 'OnePath OA IP-Platinum International Trust-EF/Sel', 'OnePath OA IP-Platinum International Trust-NEF', 'OnePath OA IP-UBS Balanced Trust-EF/Sel', 'OnePath OA IP-UBS Balanced Trust-NEF', 'OnePath OA IP-UBS Defensive Trust-EF/Sel', 'OnePath OA IP-UBS Defensive Trust-NEF', 'OnePath OA IP-UBS Diversified Fixed Income Trust-EF/Sel', 'OnePath OA IP-UBS Diversified Fixed Income Trust-NEF', 'OnePath OneAnswer Investment Portfolio - Ardea Real Outcome -EF/Sel', 'OnePath OneAnswer Investment Portfolio - Ardea Real Outcome -NE', 'OnePath OneAnswer Investment Portfolio - Barrow Hanley Concentrated Global Shares Hedged -EF/Sel', 'OnePath OneAnswer Investment Portfolio - Barrow Hanley Concentrated Global Shares Hedged -NE', 'OnePath OneAnswer Investment Portfolio - BlackRock Advantage Australian Equity -EF/Sel', 'OnePath OneAnswer Investment Portfolio - BlackRock Advantage Australian Equity -NE', 'OnePath OneAnswer Investment Portfolio - BlackRock Diversified ESG Growth -EF/Sel', 'OnePath OneAnswer Investment Portfolio - BlackRock Diversified ESG Growth -NE', 'OnePath OneAnswer Investment Portfolio - First Sentier Imputation -EF/Sel', 'OnePath OneAnswer Investment Portfolio - First Sentier Imputation -NE', 'OnePath OneAnswer Investment Portfolio - OnePath Australian Shares Index -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath Australian Shares Index -NE', 'OnePath OneAnswer Investment Portfolio - OnePath Balanced Index -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath Balanced Index -NE', 'OnePath OneAnswer Investment Portfolio - OnePath Conservative Index -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath Conservative Index -NE', 'OnePath OneAnswer Investment Portfolio - OnePath Diversified Bond Index -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath Diversified Bond Index -NE', 'OnePath OneAnswer Investment Portfolio - OnePath Growth Index -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath Growth Index -NE', 'OnePath OneAnswer Investment Portfolio - OnePath High Growth Index -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath High Growth Index -NE', 'OnePath OneAnswer Investment Portfolio - OnePath International Shares Index (Hedged) -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath International Shares Index (Hedged) -NE', 'OnePath Schroder Real Return Trust (Entry Fee)', 'OnePath Schroder Real Return Trust (Nil Entry Fee)', 'OnePath OA IP-Ausbil Australian Emerging Leaders Trust-NEF', 'Telstra Growth Pen', 'First Sentier Concentrated Aus Share', 'First Sentier Australian Small Companies', 'First Sentier Imputation', 'First Sentier Global Property Securities', 'First Sentier Australian Share', 'CFS FC-Investors Mutual Future Leaders', 'Stewart Worldwide Leaders Sustainability', 'First Sentier Property Securities', 'MyNorth Index Defensive', 'MLC MKPFPR - Altrinsic Global Eq Trust', 'MLC MKPFPR - BlackRock Global Allocation', 'MLC MKPFPR - MLC - Platinum Global Fund', 'MLC MasterKey Pension Fundamentals - Perpetual Australian Share', 'MLC MasterKey Super Fundamentals - Perpetual Australian Share', 'MLC MKPF - Perpetual WS Ethical SRI Fund', 'MLC MKSF - Perpetual WS Ethical SRI Fund', 'MLC MasterKey Pension Fundamentals (Pre Retirement) - Perpetual Smll Co Fund No.2', 'MLC MasterKey Super Fundamentals - Perpetual Small Co Fund No.2', 'MLC MKSF - PIMCO Div. Fixed Interest Wholesale Class', 'MLC MKPFPR - Platinum Asia Fund', 'MLC MKSF - Platinum Asia Fund', 'MLC MKPF - Platinum International Fund', 'MLC MKSF - Platinum International Fund', 'MLC MKPF - PM CAPITAL Global Companies', 'MLC MKSF - PM CAPITAL Global Companies', 'MLC MKPF - Schroder WS Australian Equity', 'MLC MKSF - Schroder WS Australian Equity', 'MLC MasterKey Super Fundamentals - MLC Australian Property Index', 'MLC MKSF - Vanguard Intl Shr Indx (Hgd)', 'MLC MKSF - Vanguard Intl Shr Indx', 'HOSTPLUS Fixed Interest Indexed Super', 'Australian Unity Inv Wholesale Deposits Fund', 'Lifeplan Investment Bond Lifeplan Capital Guaranteed', 'Lifeplan Investment Bond Perpetual Balanced Growth', 'Lifeplan Investment Bond Perpetual Conservative Growth', 'Lifeplan Investment Bond Perpetual Industrial Share', 'Legal Super High Growth Pen', 'Legal Super Balanced Socially Responsible Pen', 'Legal Super Growth Pen', 'Legal Super Conservative Pen', 'Legal Super Conservative Balanced Pen', 'Legal Super Balanced Index Pen', 'Legal Super Balanced Pen', 'Legal Super Cash Pen', 'Legal Super Australian Shares Pen', 'Legal Super Overseas Shares Pen', 'Legal Super Enhanced Cash Pen', 'Dimensional Australian Core Equity Trust', 'CFS FC ESup-CFS Diversified Fix Int', 'FC W Pen-CFS TTR Conservative', 'FC W Pen-CFS TTR Diversified', 'FC W Pen-CFS TTR High Growth', 'FC W Pen-CFS TTR Australian Share', 'FC W Pen-CFS TTR Property Securities', 'FC W Pen-CFS TTR Moderate', 'FC W Pen-CFS TTR Balanced', 'FC W Pen-CFS TTR Growth', 'FC W Pen-CFS TTR Australian Small Companies', 'FC W Pen-CFS TTR Global Infrastructure Securities', 'FC W Pen-CFS TTR Fixed Interest', 'FC W Pen-CFS TTR Global Share', 'FC W Pen-CFS TTR Emerging Markets', 'FC W Pen-CFS TTR Defensive', 'CFS MIF-Geared Share NEF', 'Dimensional Australia Core Equity Trust - Active ETF']\n",
|
||
"All Providers Results: \n",
|
||
"Document List File - ./sample_documents/aus_prospectus_29_documents_sample.txt\n",
|
||
"Metric \tF1-Score \tPrecision \tRecall \tAccuracy \tSUPPORT \tTP \tTN \tFP \tFN \n",
|
||
"management_fee_and_costs \t0.9529 \t0.9101 \t1.0000 \t0.9101 \t178 \t162 \t0 \t16 \t0 \n",
|
||
"management_fee \t0.9829 \t0.9663 \t1.0000 \t0.9663 \t178 \t172 \t0 \t6 \t0 \n",
|
||
"performance_fee_costs \t0.8830 \t0.8646 \t0.9022 \t0.8764 \t95 \t83 \t73 \t13 \t9 \n",
|
||
"interposed_vehicle_performance_fee_cost \t0.8814 \t0.7879 \t1.0000 \t0.9213 \t53 \t52 \t112 \t14 \t0 \n",
|
||
"administration_fees \t1.0000 \t1.0000 \t1.0000 \t1.0000 \t1 \t1 \t177 \t0 \t0 \n",
|
||
"buy_spread \t0.9856 \t0.9716 \t1.0000 \t0.9719 \t176 \t171 \t2 \t5 \t0 \n",
|
||
"sell_spread \t0.9767 \t0.9545 \t1.0000 \t0.9551 \t176 \t168 \t2 \t8 \t0 \n",
|
||
"minimum_initial_investment \t0.9611 \t0.9577 \t0.9645 \t0.9382 \t141 \t136 \t31 \t6 \t5 \n",
|
||
"benchmark_name \t0.9184 \t0.8654 \t0.9783 \t0.9101 \t100 \t90 \t72 \t14 \t2 \n",
|
||
"TOTAL \t0.9491 \t0.9198 \t0.9828 \t0.9388 \t1098 \t1035 \t469 \t82 \t94 \n",
|
||
"Total Shares Matched - 173\n",
|
||
"Total Shares Not Matched - 18\n",
|
||
"Percentage of Shares Matched - 90.57591623036649\n",
|
||
"Not Matched Shares Name List - ['Dimensional Australian Core Equity Trust', 'CFS FC ESup-CFS Diversified Fix Int', 'FC W Pen-CFS TTR Conservative', 'FC W Pen-CFS TTR Diversified', 'FC W Pen-CFS TTR High Growth', 'FC W Pen-CFS TTR Australian Share', 'FC W Pen-CFS TTR Property Securities', 'FC W Pen-CFS TTR Moderate', 'FC W Pen-CFS TTR Balanced', 'FC W Pen-CFS TTR Growth', 'FC W Pen-CFS TTR Australian Small Companies', 'FC W Pen-CFS TTR Global Infrastructure Securities', 'FC W Pen-CFS TTR Fixed Interest', 'FC W Pen-CFS TTR Global Share', 'FC W Pen-CFS TTR Emerging Markets', 'FC W Pen-CFS TTR Defensive', 'CFS MIF-Geared Share NEF', 'Dimensional Australia Core Equity Trust - Active ETF']\n",
|
||
"All Providers Results: \n",
|
||
"Document List File - ./sample_documents/aus_prospectus_17_documents_sample.txt\n",
|
||
"Metric \tF1-Score \tPrecision \tRecall \tAccuracy \tSUPPORT \tTP \tTN \tFP \tFN \n",
|
||
"management_fee_and_costs \t0.9245 \t0.8899 \t0.9619 \t0.8596 \t235 \t202 \t0 \t25 \t8 \n",
|
||
"management_fee \t0.9195 \t0.8811 \t0.9615 \t0.8511 \t235 \t200 \t0 \t27 \t8 \n",
|
||
"performance_fee_costs \t0.9345 \t0.9573 \t0.9128 \t0.9064 \t178 \t157 \t56 \t7 \t15 \n",
|
||
"interposed_vehicle_performance_fee_cost \t1.0000 \t1.0000 \t1.0000 \t1.0000 \t20 \t20 \t215 \t0 \t0 \n",
|
||
"administration_fees \t1.0000 \t1.0000 \t1.0000 \t1.0000 \t51 \t51 \t184 \t0 \t0 \n",
|
||
"total_annual_dollar_based_charges \t1.0000 \t1.0000 \t1.0000 \t1.0000 \t52 \t52 \t183 \t0 \t0 \n",
|
||
"buy_spread \t0.8693 \t0.8125 \t0.9346 \t0.8170 \t183 \t143 \t49 \t33 \t10 \n",
|
||
"sell_spread \t0.8727 \t0.8182 \t0.9351 \t0.8213 \t183 \t144 \t49 \t32 \t10 \n",
|
||
"minimum_initial_investment \t0.9842 \t1.0000 \t0.9689 \t0.9787 \t161 \t156 \t74 \t0 \t5 \n",
|
||
"benchmark_name \t0.8972 \t0.9057 \t0.8889 \t0.9532 \t55 \t48 \t176 \t5 \t6 \n",
|
||
"TOTAL \t0.9402 \t0.9265 \t0.9564 \t0.9187 \t1353 \t1173 \t986 \t129 \t156 \n",
|
||
"Total Shares Matched - 235\n",
|
||
"Total Shares Not Matched - 124\n",
|
||
"Percentage of Shares Matched - 65.45961002785515\n",
|
||
"Not Matched Shares Name List - ['SPDR® S&P World ex Australia Carbon Control Fund', 'Mercer Multi-manager Growth Fund – Retail Units', 'Mercer Multi-manager High Growth Fund – Retail Units', 'ANZ OA IP-OP Diversified Credit EF', 'ANZ OA IP-OP Diversified Credit NE', 'OnePath ANZ OA IP-T. Rowe Price Dyna Gl Bond EF', 'OnePath ANZ OA IP-T. Rowe Price Dyna Gl Bond NE', 'OnePath OA IP- Pendal Monthly Income Plus-EF/Sel', 'OnePath OA IP-ANZ Cash Advantage-EF/Sel', 'OnePath OA IP-ANZ Cash Advantage-NEF', 'OnePath OA IP-Ausbil Australian Emerging Leaders Trust-EF/Sel', 'OnePath OA IP-Bentham Global Income Trust-EF/Sel', 'OnePath OA IP-Bentham Global Income Trust-NEF', 'OnePath OA IP-Fidelity Australian Equities-EF/Sel', 'OnePath OA IP-Fidelity Australian Equities-NEF', 'OnePath OA IP-Investors Mutual Australian Share Trust- EF/Sel', 'OnePath OA IP-Investors Mutual Australian Share Trust- NEF', 'OnePath OA IP-Kapstream Absolute Return Income Trust-EF/Sel', 'OnePath OA IP-Kapstream Absolute Return Income Trust-NEF', 'OnePath OA IP-Merlon Australian Share Income-EF/Sel', 'OnePath OA IP-OnePath Active Growth Trust-NEF', 'OnePath OA IP-OnePath High Growth Trust-EF/Sel', 'OnePath OA IP-OnePath High Growth Trust-NEF', 'OnePath OA IP-OnePath Managed Growth Trust-EF/Sel', 'OnePath OA IP-OnePath Managed Growth Trust-NEF', 'OnePath OA IP-OptiMix Australian Fixed Interest Trust-EF/Sel', 'OnePath OA IP-OptiMix Australian Fixed Interest Trust-NEF', 'OnePath OA IP-OptiMix Australian Share Trust-EF/Sel', 'OnePath OA IP-OptiMix Australian Share Trust-NEF', 'OnePath OA IP-OptiMix Global Emerging Markets Share-EF/Sel', 'OnePath OA IP-OptiMix Global Emerging Markets Share-NEF', 'OnePath OA IP-OptiMIx Global Share Trust-EF/Sel', 'OnePath OA IP-OptiMIx Global Share Trust-NEF', 'OnePath OA IP-OptiMix High Growth Trust-EF/Sel', 'OnePath OA IP-OptiMix High Growth Trust-NEF', 'OnePath OA IP-OptiMix Property Securities Trust-EF/Sel', 'OnePath OA IP-OptiMix Property Securities Trust-NEF', 'OnePath OA IP-Perpetual Balanced Growth Trust-EF/Sel', 'OnePath OA IP-Perpetual Balanced Growth Trust-NEF', 'OnePath OA IP-Perpetual Conservative Growth Trust-EF/Sel', 'OnePath OA IP-Perpetual Conservative Growth Trust-NEF', 'OnePath OA IP-Platinum International Trust-EF/Sel', 'OnePath OA IP-Platinum International Trust-NEF', 'OnePath OA IP-UBS Balanced Trust-EF/Sel', 'OnePath OA IP-UBS Balanced Trust-NEF', 'OnePath OA IP-UBS Defensive Trust-EF/Sel', 'OnePath OA IP-UBS Defensive Trust-NEF', 'OnePath OA IP-UBS Diversified Fixed Income Trust-EF/Sel', 'OnePath OA IP-UBS Diversified Fixed Income Trust-NEF', 'OnePath OneAnswer Investment Portfolio - Ardea Real Outcome -EF/Sel', 'OnePath OneAnswer Investment Portfolio - Ardea Real Outcome -NE', 'OnePath OneAnswer Investment Portfolio - Barrow Hanley Concentrated Global Shares Hedged -EF/Sel', 'OnePath OneAnswer Investment Portfolio - Barrow Hanley Concentrated Global Shares Hedged -NE', 'OnePath OneAnswer Investment Portfolio - BlackRock Advantage Australian Equity -EF/Sel', 'OnePath OneAnswer Investment Portfolio - BlackRock Advantage Australian Equity -NE', 'OnePath OneAnswer Investment Portfolio - BlackRock Diversified ESG Growth -EF/Sel', 'OnePath OneAnswer Investment Portfolio - BlackRock Diversified ESG Growth -NE', 'OnePath OneAnswer Investment Portfolio - First Sentier Imputation -EF/Sel', 'OnePath OneAnswer Investment Portfolio - First Sentier Imputation -NE', 'OnePath OneAnswer Investment Portfolio - OnePath Australian Shares Index -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath Australian Shares Index -NE', 'OnePath OneAnswer Investment Portfolio - OnePath Balanced Index -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath Balanced Index -NE', 'OnePath OneAnswer Investment Portfolio - OnePath Conservative Index -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath Conservative Index -NE', 'OnePath OneAnswer Investment Portfolio - OnePath Diversified Bond Index -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath Diversified Bond Index -NE', 'OnePath OneAnswer Investment Portfolio - OnePath Growth Index -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath Growth Index -NE', 'OnePath OneAnswer Investment Portfolio - OnePath High Growth Index -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath High Growth Index -NE', 'OnePath OneAnswer Investment Portfolio - OnePath International Shares Index (Hedged) -EF/Sel', 'OnePath OneAnswer Investment Portfolio - OnePath International Shares Index (Hedged) -NE', 'OnePath Schroder Real Return Trust (Entry Fee)', 'OnePath Schroder Real Return Trust (Nil Entry Fee)', 'OnePath OA IP-Ausbil Australian Emerging Leaders Trust-NEF', 'Telstra Growth Pen', 'First Sentier Concentrated Aus Share', 'First Sentier Australian Small Companies', 'First Sentier Imputation', 'First Sentier Global Property Securities', 'First Sentier Australian Share', 'CFS FC-Investors Mutual Future Leaders', 'Stewart Worldwide Leaders Sustainability', 'First Sentier Property Securities', 'MyNorth Index Defensive', 'MLC MKPFPR - Altrinsic Global Eq Trust', 'MLC MKPFPR - BlackRock Global Allocation', 'MLC MKPFPR - MLC - Platinum Global Fund', 'MLC MasterKey Pension Fundamentals - Perpetual Australian Share', 'MLC MasterKey Super Fundamentals - Perpetual Australian Share', 'MLC MKPF - Perpetual WS Ethical SRI Fund', 'MLC MKSF - Perpetual WS Ethical SRI Fund', 'MLC MasterKey Pension Fundamentals (Pre Retirement) - Perpetual Smll Co Fund No.2', 'MLC MasterKey Super Fundamentals - Perpetual Small Co Fund No.2', 'MLC MKSF - PIMCO Div. Fixed Interest Wholesale Class', 'MLC MKPFPR - Platinum Asia Fund', 'MLC MKSF - Platinum Asia Fund', 'MLC MKPF - Platinum International Fund', 'MLC MKSF - Platinum International Fund', 'MLC MKPF - PM CAPITAL Global Companies', 'MLC MKSF - PM CAPITAL Global Companies', 'MLC MKPF - Schroder WS Australian Equity', 'MLC MKSF - Schroder WS Australian Equity', 'MLC MasterKey Super Fundamentals - MLC Australian Property Index', 'MLC MKSF - Vanguard Intl Shr Indx (Hgd)', 'MLC MKSF - Vanguard Intl Shr Indx', 'HOSTPLUS Fixed Interest Indexed Super', 'Australian Unity Inv Wholesale Deposits Fund', 'Lifeplan Investment Bond Lifeplan Capital Guaranteed', 'Lifeplan Investment Bond Perpetual Balanced Growth', 'Lifeplan Investment Bond Perpetual Conservative Growth', 'Lifeplan Investment Bond Perpetual Industrial Share', 'Legal Super High Growth Pen', 'Legal Super Balanced Socially Responsible Pen', 'Legal Super Growth Pen', 'Legal Super Conservative Pen', 'Legal Super Conservative Balanced Pen', 'Legal Super Balanced Index Pen', 'Legal Super Balanced Pen', 'Legal Super Cash Pen', 'Legal Super Australian Shares Pen', 'Legal Super Overseas Shares Pen', 'Legal Super Enhanced Cash Pen']\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"\n",
|
||
"\"\"\"\n",
|
||
"Blade's updates\n",
|
||
"1. Set the secondary key to be the share class name, instead of the fund name\n",
|
||
"2. Remove the data point which support is 0 to calculate the metrics\n",
|
||
"3. Add the message list to store the error message\n",
|
||
"4. Support save metrics/ error message to excel file\n",
|
||
"5. Support statistics for different document list\n",
|
||
"6. Set F1-Score to the first column in the metrics table\n",
|
||
"\"\"\"\n",
|
||
"\n",
|
||
"# Load the files\n",
|
||
"headers_gt, ground_truth_data = load_excel(path_ground_truth, 0)\n",
|
||
"headers_gen, generated_results_data = load_excel(path_generated_results, 0)\n",
|
||
"\n",
|
||
"# Assuming doc_id is the first column and fund_name is the second column\n",
|
||
"# doc_id_index = 0\n",
|
||
"# fund_name_index = 1\n",
|
||
"\n",
|
||
"# Index the data\n",
|
||
"ground_truth_indexed = index_data_by_key(ground_truth_data, headers_gt)\n",
|
||
"generated_results_indexed = index_data_by_key(generated_results_data, headers_gen)\n",
|
||
"\n",
|
||
"intersection = set(headers_gen).intersection(headers_gt)\n",
|
||
"\n",
|
||
"# Convert the result back to a list (if you need it as a list)\n",
|
||
"intersection_list = list(intersection)\n",
|
||
"\n",
|
||
"total_fn = []\n",
|
||
"\n",
|
||
"# df_provider_mapping = get_provider_mapping(provider_mapping_file_path)\n",
|
||
"\n",
|
||
"# all_provider_dict = get_provider_names(generated_results_indexed, df_provider_mapping)\n",
|
||
"\n",
|
||
"\n",
|
||
"# for provider_name in all_provider_dict:\n",
|
||
"# provider_vise_generated_results = get_specified_doc_data(generated_results_indexed, all_provider_dict[provider_name])\n",
|
||
"# comparison_results, funds_matched, funds_not_matched = compare_data(ground_truth_indexed, provider_vise_generated_results, headers_gt, doc_id_index, fund_name_index, intersection_list,funds_matched, funds_not_matched)\n",
|
||
"# print(\"\\n\")\n",
|
||
"# print(\"\\n\")\n",
|
||
"# print(\"Provider Name - \" + provider_name + \"\\t Number of Docs - \" + str(len(all_provider_dict[provider_name])))\n",
|
||
"# #create_metrics_df(comparison_results)\n",
|
||
"# print_metrics_table(comparison_results)\n",
|
||
"# print(\"Total Funds Matched - \" + str(funds_matched) + \"\\nTotal Funds Not Matched - \" + str(funds_not_matched))\n",
|
||
"# print(\"Percentage of Funds Matched - \" + str((funds_matched/(funds_matched + funds_not_matched))*100))\n",
|
||
"\n",
|
||
"\n",
|
||
"\n",
|
||
"print(\"\\n\")\n",
|
||
"print(\"\\n\")\n",
|
||
"document_list_file_list = [None, \n",
|
||
" \"./sample_documents/aus_prospectus_29_documents_sample.txt\", \n",
|
||
" \"./sample_documents/aus_prospectus_17_documents_sample.txt\"\n",
|
||
" ]\n",
|
||
"# document_list_file_list = [None]\n",
|
||
"for document_list_file in document_list_file_list:\n",
|
||
" document_list = None\n",
|
||
" if document_list_file is not None:\n",
|
||
" with open(document_list_file, \"r\", encoding=\"utf-8\") as f:\n",
|
||
" document_list = f.readlines()\n",
|
||
" document_list = [doc_id.strip() for doc_id in document_list]\n",
|
||
" \n",
|
||
" print(\"All Providers Results: \")\n",
|
||
" print(\"Document List File - \", document_list_file)\n",
|
||
" comparison_results, message_list, share_matched, \\\n",
|
||
" share_not_matched, not_matched_share_name_list = compare_data(ground_truth_indexed, \n",
|
||
" generated_results_indexed,\n",
|
||
" headers_gt,\n",
|
||
" intersection_list,\n",
|
||
" document_list)\n",
|
||
" metrics_list = print_metrics_table(comparison_results)\n",
|
||
" print(\"Total Shares Matched - \" + str(share_matched) + \"\\nTotal Shares Not Matched - \" + str(share_not_matched))\n",
|
||
" print(\"Percentage of Shares Matched - \" + str((share_matched/(share_matched + share_not_matched))*100))\n",
|
||
" print(\"Not Matched Shares Name List - \", not_matched_share_name_list)\n",
|
||
"\n",
|
||
" metrics_df = pd.DataFrame(metrics_list)\n",
|
||
" message_df = pd.DataFrame(message_list)\n",
|
||
" share_matched_data = {\"share_matched\": share_matched, \"share_not_matched\": share_not_matched, \"not_matched_share_name_list\": not_matched_share_name_list}\n",
|
||
" share_matched_df = pd.DataFrame([share_matched_data])\n",
|
||
"\n",
|
||
" output_metrics_folder = r\"/data/aus_prospectus/output/metrics_data/\"\n",
|
||
" os.makedirs(output_metrics_folder, exist_ok=True)\n",
|
||
" if os.path.exists(output_metrics_folder):\n",
|
||
" generated_file_base_name = os.path.basename(path_generated_results).replace(\".xlsx\", \"\")\n",
|
||
" metrics_file_name = f\"metrics_{generated_file_base_name}\"\n",
|
||
" if document_list_file is not None:\n",
|
||
" metrics_file_name = f\"{metrics_file_name}_{len(document_list)}_documents.xlsx\"\n",
|
||
" else:\n",
|
||
" metrics_file_name = f\"{metrics_file_name}_all_documents.xlsx\"\n",
|
||
" metrics_file_path = os.path.join(output_metrics_folder, metrics_file_name)\n",
|
||
" with pd.ExcelWriter(metrics_file_path) as writer:\n",
|
||
" metrics_df.to_excel(writer, sheet_name=\"metrics_data\", index=False)\n",
|
||
" message_df.to_excel(writer, sheet_name=\"message_data\", index=False)\n",
|
||
" share_matched_df.to_excel(writer, sheet_name=\"share_matched_data\", index=False)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"for message_list_element in message_list:\n",
|
||
" if message_list_element[\"data_point\"] == \"performance_fee_costs\":\n",
|
||
" print(message_list_element)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import pandas as pd\n",
|
||
"\n",
|
||
"\n",
|
||
"# Convert data to DataFrame\n",
|
||
"df = pd.DataFrame(message_list)\n",
|
||
"\n",
|
||
"# Sort DataFrame by 'doc_id'\n",
|
||
"df_sorted = df.sort_values(by=['doc_id'])\n",
|
||
"\n",
|
||
"# Save DataFrame to Excel file\n",
|
||
"os.makedirs(\"/data/aus_prospectus/output/error_analysis/\", exist_ok=True)\n",
|
||
"output_filename = r\"/data/aus_prospectus/output/error_analysis/anomalies_found.xlsx\"\n",
|
||
"df_sorted.to_excel(output_filename, index=False)\n",
|
||
"\n",
|
||
"print(f\"Excel file '{output_filename}' has been created successfully.\")\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "emea_ar_test",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.12.6"
|
||
},
|
||
"orig_nbformat": 4
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 2
|
||
}
|