blade
/
dc-ml-emea-ar
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

a little change

This commit is contained in:
Blade He 2024-09-16 16:43:03 -05:00
parent 932870f406
commit 50e6c3c19d
2 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/data_extraction.py
View File

@ -253,6 +253,7 @@ class DataExtraction:
exclude_data: list) -> list:
"""
If occur error, split the context to two parts and try to get data from the two parts
Relevant document: 503194284
"""
try:
logger.info(f"Split context to get data to fix issue which output length is over 4K tokens")

2
main.py
View File

@ -505,7 +505,7 @@ if __name__ == "__main__":
# doc_id = "476492237"
# extract_data(doc_id, pdf_folder, output_extract_data_child_folder, re_run)
special_doc_id_list = ["503194284"]
special_doc_id_list = ["508854243"]
output_mapping_child_folder = r"/data/emea_ar/output/mapping_data/docs/"
output_mapping_total_folder = r"/data/emea_ar/output/mapping_data/total/"
re_run_mapping_data = True