gctools/misc/csv_cleanup.py

import plac
import re
from pathlib import Path
from datetime import datetime


MIN_AGE = datetime.now().timestamp() - 12 * 60 * 60


def csv_cleanup(dirname: str = "misc/data"):
    for csv_file in Path(dirname).glob("*.csv"):
        temp_file = Path(str(csv_file) + ".tmp")
        if csv_file.stat().st_mtime < MIN_AGE:
            continue
        print(csv_file.name)

        with (
            open(csv_file, "r", encoding="latin-1") as frh,
            open(temp_file, "w", encoding="latin-1") as fwh,
        ):
            buffer = " "
            while buffer != "":
                buffer = frh.read(10_000)
                fwh.write(re.sub(r'(?<!")\r?\n', "", buffer))

        csv_file.unlink()
        temp_file.rename(csv_file)


if __name__ == "__main__":
    plac.call(csv_cleanup)