File tree Expand file tree Collapse file tree 1 file changed +4
-4
lines changed Expand file tree Collapse file tree 1 file changed +4
-4
lines changed Original file line number Diff line number Diff line change @@ -91,19 +91,19 @@ def get_wikitext2_text(tokenizer):
9191 return test_enc , test_text
9292
9393
94- def get_flores_plus_text (tokenizer , lng_scrpt ):
94+ def get_flores_plus_text (tokenizer , lng_script ):
9595 hf_hub_download (
9696 repo_id = "alexei-v-ivanov-amd/flores_plus" ,
9797 repo_type = "dataset" ,
98- filename = lng_scrpt + ".parquet" ,
98+ filename = lng_script + ".parquet" ,
9999 local_dir = "./" ,
100100 )
101101
102- df = pandas .read_parquet ("./" + lng_scrpt + ".parquet" )
102+ df = pandas .read_parquet ("./" + lng_script + ".parquet" )
103103 test_text = "\n \n " .join (line .strip () for line in df ["text" ])
104104 test_enc = tokenizer (test_text )
105105
106- os .remove ("./" + lng_scrpt + ".parquet" )
106+ os .remove ("./" + lng_script + ".parquet" )
107107
108108 return test_enc , test_text
109109
You can’t perform that action at this time.
0 commit comments