sqlalchemy mit zentralem Aufruf. db_run angepasst

database/db_compare.py

@@ -3,8 +3,7 @@ import json
 from pathlib import Path
 
 import pandas as pd
-import pyodbc
-from sqlalchemy import create_engine
+from pyodbc import ProgrammingError
 
 from database.db_create import get_import_config
 from database.model import DatabaseInspect, DbCreateConfig, create_db_ini, load_config
@@ -79,7 +78,7 @@ def compare(config_file: str = "database/CARLO.json"):
                 try:
                     q = source_db.cursor.execute(query_ts)
                     source_row_count_ts[client_db] = q.fetchone()[0]
-                except pyodbc.ProgrammingError:
+                except ProgrammingError:
                     pass
 
             if dest_row_count.get(client_db, 0) != source_row_count.get(client_db, 0):
@@ -115,12 +114,12 @@ def compare_details(
     )
 
     source_file = f"{cfg.stage_dir}\\source\\{table_client}.csv"
-    source_data = pd.read_sql(query_source, create_engine(source_db.conn_string_sqlalchemy))
+    source_data = pd.read_sql(query_source, source_db.sqlalchemy_engine)
     source_data["timestamp"] = source_data["timestamp"].apply(decode_ts)
     source_data.to_csv(source_file, index=False)
 
     dest_file = f"{cfg.stage_dir}\\dest\\{table_client}.csv"
-    dest_data = pd.read_sql(query_dest, create_engine(dest_db.conn_string_sqlalchemy))
+    dest_data = pd.read_sql(query_dest, dest_db.sqlalchemy_engine)
     dest_data["timestamp"] = dest_data["timestamp"].apply(decode_ts)
     dest_data.to_csv(dest_file, index=False)
 

database/db_create.py

@@ -166,6 +166,14 @@ def create(config_file: str = "database/CARLO.json"):
             f.write(f"echo {current_table['dest']} >CON\n")
             f.write(f"call {cfg.batch_dir}\\{current_table['dest']}.bat 1\n\n")
 
+    with open(f"{cfg.batch_dir}/_{cfg.name}_full_load.bat", "w", encoding="cp850") as f:
+        f.write("@echo off & cd /d %~dp0\n")
+        f.write(f"del {cfg.stage_dir}\\*.* /Q /F >nul 2>nul\n\n")
+        for index, current_table in config.iterrows():
+            f.write(f"echo =={current_table['dest']}==\n")
+            f.write(f"echo {current_table['dest']} >CON\n")
+            f.write(f"call {cfg.batch_dir}\\{current_table['dest']}.bat\n\n")
+
 
 if __name__ == "__main__":
     create()

database/db_run.py

@@ -12,11 +12,10 @@ def task(name: str, increment: bool = True) -> subprocess.Popen:
     return subprocess.Popen(f'C:\\Windows\\System32\\cmd.exe /C "{name} {flag}"', stdout=logfile.open("w")).wait()
 
 
-def run(config_file: str, increment: int, max: int) -> None:
+def run(config_file: str, increment: bool, max: int) -> None:
     cfg = load_config(config_file)
     files = [str(f) for f in Path(cfg.batch_dir).glob("*.bat") if not f.name.startswith("_")]
-    flag = increment == 1
-    task2 = partial(task, increment=flag)
+    task2 = partial(task, increment=increment)
 
     with ThreadPoolExecutor(max_workers=max) as executor:
         executor.map(task2, files)

database/model.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 
 import pyodbc
+from sqlalchemy import create_engine
 
 
 @dataclass
@@ -99,6 +100,18 @@ class DatabaseInspect:
     def bcp_conn_params(self):
         return f"-S {self.dsn.server} -d {self.dsn.database} -U {self.dsn.user} -P {self.dsn.password}"
 
+    @property
+    def cursor(self):
+        if not self._cursor:
+            self._cursor = self.connect()
+        return self._cursor
+
+    @property
+    def sqlalchemy_engine(self):
+        if not self._sqlalchemy_engine:
+            self._sqlalchemy_engine = create_engine(self.conn_string_sqlalchemy)
+        return self._sqlalchemy_engine
+
     def connect(self):
         c = pyodbc.connect(self.conn_string)
         return c.cursor()

db.py

@@ -20,9 +20,9 @@ def compare(config_file: str):
 
 
 @app.command()
-def run(config_file: str, increment: str = 1, max: int = 5):
+def run(config_file: str, increment: str = "1", max: int = 5):
     config_file = cfg.system_dir + f"\\SQL\\config\\{config_file}.json"
-    database.run(config_file, increment, max)
+    database.run(config_file, increment == "1", max)
 
 
 @app.command()