Merge pull request #53 from TogetherCrew/feat/mediawiki-batch-ingestion

feat: added max 10 parallel processing!

Author: amindadgar

hivemind_etl/mediawiki/etl.py

@@ -106,7 +106,7 @@ def load(self, documents: list[Document]) -> None:
         batch_size = 1000
         batches = [documents[i:i + batch_size] for i in range(0, len(documents), batch_size)]
 
-        with ThreadPoolExecutor() as executor:
+        with ThreadPoolExecutor(max_workers=10) as executor:
             # Submit all batch processing tasks
             future_to_batch = {
                 executor.submit(ingestion_pipeline.run_pipeline, batch): i