ft-orchestration jq.py use parallel with --linebuffer and prevent ram exhaustion

Author: pascal260303

tools/ft-orchestration/src/collector/jq.py

@@ -121,7 +121,7 @@ def __init__(self, executor: Executor, file: str):
 EOF
 
 ipfixcol2 -c {Path(self._conf_dir, self.CONFIG_FILE)} |
-parallel --pipe --block 10M --recend "}}" --recstart "{{" -j 0 -- jq -r -f "$file"
+parallel --pipe --recend '\n' --line-buffer --memfree 6G -L 200K --retries 0 -j 100% -- jq -r -f "$file"
 """
         # Reads fds file and outputs json with ipfixcol2, then converts json with jq to csv.
         # In the csv output the IPv4/IPv6 addresses are merged to source/destinationIPAddress.
@@ -351,4 +351,3 @@ def save_csv(csv_file: str):
         except Exception:  # noqa: BLE001 - best-effort cleanup
             pass
         logging.getLogger().info("CSV output saved in %.2f seconds.", (end - start))
-        logging.getLogger().info("CSV output saved in %.2f seconds.", (end - start))