bcp csv-trim

misc/csv_trim.py

@@ -0,0 +1,31 @@
+import os
+import re
+from datetime import datetime
+from pathlib import Path
+
+MAX_AGE = datetime.now().timestamp() - 12 * 60 * 60
+
+
+def csv_trim(dirname: str = "misc/data"):
+    if Path(dirname).is_file():
+        csv_trim_file(Path(dirname), True)
+    else:
+        for csv_file in Path(dirname).glob("*.csv"):
+            csv_trim_file(csv_file)
+
+
+def csv_trim_file(csv_file: Path, ignore_age: bool = False):
+    temp_file = Path(str(csv_file) + ".tmp")
+    file_mtime = csv_file.stat().st_mtime
+    if not ignore_age and file_mtime < MAX_AGE:
+        return
+    print(csv_file.name)
+    with open(csv_file, "r", encoding="latin-1", errors="ignore") as frh:
+        with open(temp_file, "w", encoding="latin-1") as fwh:
+            for line in frh.readlines():
+                fwh.write(re.sub(r"[ ]+\t", "\t", line))
+
+    os.utime(temp_file, (file_mtime, file_mtime))
+
+    csv_file.unlink()
+    temp_file.rename(csv_file)

misc2.py

@@ -6,6 +6,7 @@ import config
 from misc import apache_ldap
 from misc.convert_mail_csv import convert_folder_to_combined_csv
 from misc.csv_cleanup import csv_cleanup
+from misc.csv_trim import csv_trim
 from misc.datev_status import print_all_mdb_files
 from misc.headers import create_headerfiles
 
@@ -51,6 +52,13 @@ def csv_check(path: str = ""):
     csv_cleanup(path)
 
 
+@app.command()
+def bcp_csv_trim(path: str = ""):
+    if path == "":
+        path = f"{cfg.system_dir}\\SQL\\temp"
+    csv_trim(path)
+
+
 @app.command()
 def combine_csv(path: str = ""):
     if path == "":