DatabaseTools als db.exe (erster Versuch)

.gitignore

@@ -1,2 +1,4 @@
 __pycache__
-build/
+build/
+venv/
+.venv/

database/__init__.py

@@ -0,0 +1,3 @@
+from database.db_create import create
+from database.db_run import run
+from database.db_schema import schema

database/db_create.py

@@ -1,31 +1,49 @@
 import json
 from collections import namedtuple
 from pathlib import Path
-from re import escape
-from numpy import select
 import pandas as pd
 import pyodbc
-from dataclasses import dataclass
 
-
-DbCreateConfig = namedtuple('DbCreateConfig', 'name csv_file clients filter source_dsn target_dsn stage_dir batch_dir')
-DsnConfig = namedtuple('DsnConfig', 'user password server database driver schema')
-
-cfg = DbCreateConfig(**{
-    'name': 'CARLO',
-    'csv_file': 'CARLO.csv',
-    'clients': {'1': 'M und S Fahrzeughandel GmbH'},
-    'filter': ['01.01.2018', '01.01.2019'],
-    'source_dsn': {'user': 'sa', 'password': 'Mffu3011#', 'server': 'GC-SERVER1\\GLOBALCUBE',
-                   'database': 'DE0017', 'driver': 'mssql', 'schema': 'dbo'},
-    'target_dsn': {'user': 'sa', 'password': 'Mffu3011#', 'server': 'GC-SERVER1\\GLOBALCUBE',
-                   'database': 'CARLO2', 'driver': 'mssql', 'schema': 'import'},
-    'stage_dir': '..\\temp',
-    'batch_dir': '..\\batch'
-})
-
-
-class database_inspect():
+# from re import escape
+# from numpy import select
+# from dataclasses import dataclass
+
+
+DbCreateConfig = namedtuple(
+    "DbCreateConfig",
+    "name csv_file clients filter source_dsn target_dsn stage_dir batch_dir",
+)
+DsnConfig = namedtuple("DsnConfig", "user password server database driver schema")
+
+cfg = DbCreateConfig(
+    **{
+        "name": "CARLO",
+        "csv_file": "CARLO.csv",
+        "clients": {"1": "M und S Fahrzeughandel GmbH"},
+        "filter": ["01.01.2018", "01.01.2019"],
+        "source_dsn": {
+            "user": "sa",
+            "password": "Mffu3011#",
+            "server": "GC-SERVER1\\GLOBALCUBE",
+            "database": "DE0017",
+            "driver": "mssql",
+            "schema": "dbo",
+        },
+        "target_dsn": {
+            "user": "sa",
+            "password": "Mffu3011#",
+            "server": "GC-SERVER1\\GLOBALCUBE",
+            "database": "CARLO2",
+            "driver": "mssql",
+            "schema": "import",
+        },
+        "stage_dir": "..\\temp",
+        "batch_dir": "..\\batch",
+    }
+)
+
+
+class database_inspect:
     tables = []
 
     def __init__(self, dsn):
@@ -33,10 +51,12 @@ class database_inspect():
         self.cursor = self.connect()
 
     def conn_string(self):
-        if self.dsn.driver == 'mssql':
-            return 'Driver={SQL Server Native Client 11.0};' + \
-                   f"Server={self.dsn.server};Database={self.dsn.database};Uid={self.dsn.user};Pwd={self.dsn.password}"
-        if self.dsn.driver == 'mysql':
+        if self.dsn.driver == "mssql":
+            return (
+                "Driver={SQL Server Native Client 11.0};"
+                + f"Server={self.dsn.server};Database={self.dsn.database};Uid={self.dsn.user};Pwd={self.dsn.password}"
+            )
+        if self.dsn.driver == "mysql":
             return f"mysql+pymysql://{self.dsn.user}:{self.dsn.password}@{self.dsn.server}/{self.dsn.database}?charset=utf8mb4"
         return f"DSN={self.dsn.server};UID={self.dsn.user};PWD={self.dsn.password}"
 
@@ -48,48 +68,58 @@ class database_inspect():
         return c.cursor()
 
     def get_tables(self):
-        tables = [x[2] for x in self.cursor.tables(tableType='TABLE')]
-        views = [x[2] for x in self.cursor.tables(tableType='VIEW')]
+        tables = [x[2] for x in self.cursor.tables(tableType="TABLE")]
+        views = [x[2] for x in self.cursor.tables(tableType="VIEW")]
         self.tables = tables + views
         return self.tables
 
     def get_prefix(self):
         if (len(self.tables)) == 0:
             self.get_tables()
-        source_tables_prefix = dict(enumerate(sorted(list(set([t.split('$')[0] for t in self.tables if '$' in t]))), 1))
+        source_tables_prefix = dict(
+            enumerate(
+                sorted(list(set([t.split("$")[0] for t in self.tables if "$" in t]))), 1
+            )
+        )
         if len(source_tables_prefix) == 0:
-            q = self.cursor.execute('select name FROM sys.databases')
+            q = self.cursor.execute("select name FROM sys.databases")
             source_tables_prefix = [x[0] for x in q.fetchall()]
         return source_tables_prefix
 
     def get_columns(self, table):
         source_insp_cols = [col.column_name for col in self.cursor.columns(table=table)]
         if len(source_insp_cols) == 0:
-            q = self.cursor.execute('SELECT COLUMN_NAME as column_name FROM information_schema.columns ' +
-                                    f"WHERE TABLE_NAME = '{self.convert_table(table)}'")
+            q = self.cursor.execute(
+                "SELECT COLUMN_NAME as column_name FROM information_schema.columns "
+                + f"WHERE TABLE_NAME = '{self.convert_table(table)}'"
+            )
             source_insp_cols = [col[0] for col in q.fetchall()]
         return source_insp_cols
 
     def convert_table(self, table):
-        if '.' in table:
-            table = table.split('.')[-1]
-        if '[' in table:
+        if "." in table:
+            table = table.split(".")[-1]
+        if "[" in table:
             table = table[1:-1]
         return table
 
 
-def create(config_file='dbtools/OPTIMA.json'):
-    cfg_import = json.load(open(config_file, 'r', encoding='latin-1'))
+def create(config_file="dbtools/OPTIMA.json"):
+    cfg_import = json.load(open(config_file, "r", encoding="latin-1"))
     base_dir = Path(config_file).resolve().parent
-    cfg_import['name'] = Path(config_file).stem
-    if cfg_import['stage_dir'][:2] == '..':
-        cfg_import['stage_dir'] = str(base_dir.joinpath(cfg_import['stage_dir']).resolve())
-    if cfg_import['batch_dir'][:2] == '..':
-        cfg_import['batch_dir'] = str(base_dir.joinpath(cfg_import['batch_dir']).resolve())
+    cfg_import["name"] = Path(config_file).stem
+    if cfg_import["stage_dir"][:2] == "..":
+        cfg_import["stage_dir"] = str(
+            base_dir.joinpath(cfg_import["stage_dir"]).resolve()
+        )
+    if cfg_import["batch_dir"][:2] == "..":
+        cfg_import["batch_dir"] = str(
+            base_dir.joinpath(cfg_import["batch_dir"]).resolve()
+        )
     cfg = DbCreateConfig(**cfg_import)
 
-    df = pd.read_csv(f"{base_dir}/{cfg.csv_file}", sep=';', encoding='latin-1')
-    config = df[df['target'].notnull()]
+    df = pd.read_csv(f"{base_dir}/{cfg.csv_file}", sep=";", encoding="latin-1")
+    config = df[df["target"].notnull()]
     print(config.head())
 
     source_db = database_inspect(cfg.source_dsn)
@@ -100,31 +130,41 @@ def create(config_file='dbtools/OPTIMA.json'):
     target_tables = target_db.get_tables()
 
     for index, current_table in config.iterrows():
-        with open(f"{cfg.batch_dir}/{current_table['target']}.bat", 'w', encoding='cp850') as f:
-            f.write('@echo off \n')
-            f.write('rem ==' + current_table['target'] + '==\n')
-
-            if not current_table['target'] in target_tables:
-                f.write(f"echo Ziel-Tabelle '{current_table['target']}' existiert nicht!\n")
+        with open(
+            f"{cfg.batch_dir}/{current_table['target']}.bat", "w", encoding="cp850"
+        ) as f:
+            f.write("@echo off \n")
+            f.write("rem ==" + current_table["target"] + "==\n")
+
+            if not current_table["target"] in target_tables:
+                f.write(
+                    f"echo Ziel-Tabelle '{current_table['target']}' existiert nicht!\n"
+                )
                 print(f"Ziel-Tabelle '{current_table['target']}' existiert nicht!")
                 continue
 
-            f.write(f"del {cfg.stage_dir}\\{current_table['target']}*.* /Q /F >nul 2>nul \n")
-            f.write(f"sqlcmd.exe {target_db.bcp_conn_params()} -p " +
-                    f"-Q \"TRUNCATE TABLE [{cfg.target_dsn['schema']}].[{current_table['target']}]\" \n")
-
-            target_columns_list = target_db.get_columns(current_table['target'])
-            if 'CLIENT_DB' in target_columns_list:
-                target_columns_list.remove('CLIENT_DB')
-                target_columns_list.append('Client_DB')
+            f.write(
+                f"del {cfg.stage_dir}\\{current_table['target']}*.* /Q /F >nul 2>nul \n"
+            )
+            f.write(
+                f"sqlcmd.exe {target_db.bcp_conn_params()} -p "
+                + f"-Q \"TRUNCATE TABLE [{cfg.target_dsn['schema']}].[{current_table['target']}]\" \n"
+            )
+
+            target_columns_list = target_db.get_columns(current_table["target"])
+            if "CLIENT_DB" in target_columns_list:
+                target_columns_list.remove("CLIENT_DB")
+                target_columns_list.append("Client_DB")
             target_columns = set(target_columns_list)
 
             for client_db, prefix in cfg.clients.items():
-                source_table = current_table['source'].format(prefix)
+                source_table = current_table["source"].format(prefix)
                 if source_table not in source_tables:
                     source_table2 = source_db.convert_table(source_table)
                     if source_table2 not in source_tables:
-                        f.write(f"echo Quell-Tabelle '{source_table}' existiert nicht!\n")
+                        f.write(
+                            f"echo Quell-Tabelle '{source_table}' existiert nicht!\n"
+                        )
                         print(f"Quell-Tabelle '{source_table}' existiert nicht!")
                         continue
 
@@ -136,52 +176,64 @@ def create(config_file='dbtools/OPTIMA.json'):
                 if len(diff1) > 0:
                     f.write("rem Nur in Quelle: " + ";".join(diff1) + "\n")
                 diff2 = target_columns.difference(source_columns)
-                if 'Client_DB' not in diff2:
+                if "Client_DB" not in diff2:
                     f.write("echo Spalte 'Client_DB' fehlt!\n")
-                    print(f"Ziel-Tabelle '{current_table['target']}' Spalte 'Client_DB' fehlt!")
+                    print(
+                        f"Ziel-Tabelle '{current_table['target']}' Spalte 'Client_DB' fehlt!"
+                    )
                     continue
-                diff2.remove('Client_DB')
+                diff2.remove("Client_DB")
                 if len(diff2) > 0:
                     f.write("rem Nur in Ziel:   " + ";".join(diff2) + "\n")
 
-                if not pd.isnull(current_table['query']):
-                    select_query = current_table['query'].format(prefix, cfg.filter[0], cfg.filter[1])
-                elif '.' in source_table or cfg.source_dsn['schema'] == '':
-                    select_query = f"SELECT T1.* FROM \\\"{source_table}\\\" T1 "
+                if not pd.isnull(current_table["query"]):
+                    select_query = current_table["query"].format(
+                        prefix, cfg.filter[0], cfg.filter[1]
+                    )
+                elif "." in source_table or cfg.source_dsn["schema"] == "":
+                    select_query = f'SELECT T1.* FROM \\"{source_table}\\" T1 '
                 else:
                     select_query = f"SELECT T1.* FROM [{cfg.source_dsn['schema']}].[{source_table}] T1 "
 
-                if not pd.isnull(current_table['filter']):
-                    select_query += " WHERE " + current_table['filter'].format("", cfg.filter[0], cfg.filter[1])
+                if not pd.isnull(current_table["filter"]):
+                    select_query += " WHERE " + current_table["filter"].format(
+                        "", cfg.filter[0], cfg.filter[1]
+                    )
                 # select_columns = "T1.[" + "], T1.[".join(intersect) + "],"
-                select_columns = ''
+                select_columns = ""
                 for col in target_columns_list:
                     if col in intersect:
                         select_columns += f"T1.[{col}], "
-                    elif col == 'Client_DB':
-                        select_columns += "'" + client_db + "' as \\\"Client_DB\\\", "
+                    elif col == "Client_DB":
+                        select_columns += "'" + client_db + '\' as \\"Client_DB\\", '
                     else:
-                        select_columns += "'' as \\\"" + col + "\\\", "
+                        select_columns += "'' as \\\"" + col + '\\", '
 
                 select_query = select_query.replace("T1.*", select_columns[:-2])
-                select_query = select_query.replace("%", "%%")     # batch-Problem
+                select_query = select_query.replace("%", "%%")  # batch-Problem
 
-                stage_csv = f"{cfg.stage_dir}\\{current_table['target']}_{client_db}.csv"
+                stage_csv = (
+                    f"{cfg.stage_dir}\\{current_table['target']}_{client_db}.csv"
+                )
                 # insert_query = f"LOAD DATA INFILE '{stage_csv}' INTO TABLE {current_table['target']} FIELDS TERMINATED BY ','
                 # ENCLOSED BY '\"' LINES TERMINATED BY '\n';"
 
                 # print(select_query)
-                bulk_copy = 'bcp' if cfg.source_dsn['driver'] == 'mssql' else 'cet'
-                f.write(f"{bulk_copy} \"{select_query}\" queryout \"{stage_csv}\" {source_db.bcp_conn_params()} -c -C 65001 -m 1000 " +
-                        f"-e \"{stage_csv[:-4]}.queryout.log\" > \"{stage_csv[:-4]}.bcp1.log\" \n")
-                f.write(f"type \"{stage_csv[:-4]}.bcp1.log\" | findstr -v \"1000\" \n")
-                f.write(f"bcp [{cfg.target_dsn['schema']}].[{current_table['target']}] in \"{stage_csv}\" {target_db.bcp_conn_params()} " +
-                        f"-c -C 65001 -m 1000 -e \"{stage_csv[:-4]}.in.log\" > \"{stage_csv[:-4]}.bcp2.log\" \n")
-                f.write(f"type \"{stage_csv[:-4]}.bcp2.log\" | findstr -v \"1000\" \n")
-
-                f.write(f"del \"{stage_csv}\" /F >nul 2>nul \n")
-
-    with open(f"{cfg.batch_dir}/_{cfg.name}.bat", 'w', encoding='cp850') as f:
+                bulk_copy = "bcp" if cfg.source_dsn["driver"] == "mssql" else "cet"
+                f.write(
+                    f'{bulk_copy} "{select_query}" queryout "{stage_csv}" {source_db.bcp_conn_params()} -c -C 65001 -m 1000 '
+                    + f'-e "{stage_csv[:-4]}.queryout.log" > "{stage_csv[:-4]}.bcp1.log" \n'
+                )
+                f.write(f'type "{stage_csv[:-4]}.bcp1.log" | findstr -v "1000" \n')
+                f.write(
+                    f"bcp [{cfg.target_dsn['schema']}].[{current_table['target']}] in \"{stage_csv}\" {target_db.bcp_conn_params()} "
+                    + f'-c -C 65001 -m 1000 -e "{stage_csv[:-4]}.in.log" > "{stage_csv[:-4]}.bcp2.log" \n'
+                )
+                f.write(f'type "{stage_csv[:-4]}.bcp2.log" | findstr -v "1000" \n')
+
+                f.write(f'del "{stage_csv}" /F >nul 2>nul \n')
+
+    with open(f"{cfg.batch_dir}/_{cfg.name}.bat", "w", encoding="cp850") as f:
         f.write("@echo off & cd /d %~dp0 \n")
         f.write(f"del {cfg.stage_dir}\\*.* /Q /F >nul 2>nul \n\n")
         for index, current_table in config.iterrows():
@@ -190,5 +242,5 @@ def create(config_file='dbtools/OPTIMA.json'):
             f.write(f"call {current_table['target']}.bat\n\n")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     create()

database/db_run.py

@@ -4,15 +4,17 @@ import subprocess
 
 
 def task(name):
-    return subprocess.Popen(f'C:\\Windows\\System32\\cmd.exe /C "{name}"', stdout=subprocess.DEVNULL).wait()
+    return subprocess.Popen(
+        f'C:\\Windows\\System32\\cmd.exe /C "{name}"', stdout=subprocess.DEVNULL
+    ).wait()
 
 
 def run(base_dir):
-    files = [str(f) for f in Path(base_dir).glob('*.bat') if not f.name.startswith('_')]
+    files = [str(f) for f in Path(base_dir).glob("*.bat") if not f.name.startswith("_")]
 
     with ThreadPoolExecutor(max_workers=5) as executor:
         executor.map(task, files)
 
 
-if __name__ == '__main__':
-    run('C:\\GlobalCube\\System\\OPTIMA\\SQL\\batch')
+if __name__ == "__main__":
+    run("C:\\GlobalCube\\System\\OPTIMA\\SQL\\batch")

database/db_schema.py

@@ -9,25 +9,27 @@ def connect():
 
 def convert_desc(col):
     nullable = "NULL" if col.nullable == 1 else "NOT NULL"
-    if col.type_name == 'CHAR':
-        return f'   [{col.column_name}] [varchar]({col.length}) {nullable}'
-    if col.type_name == 'HEX':
-        return f'   [{col.column_name}] [binary]({col.length}) {nullable}'
-    if col.type_name == 'INTEGER':
-        return f'   [{col.column_name}] [int] {nullable}'
-    if col.type_name == 'DATE':
-        return f'   [{col.column_name}] [datetime] {nullable}'
-    if col.type_name == 'NUMERIC':
-        return f'   [{col.column_name}] [decimal]({col.length},{col.scale}) {nullable}'
-    if col.type_name == 'BIT':
-        return f'   [{col.column_name}] [boolean] {nullable}'
-    if col.type_name == 'MEMO':
-        return f'   [{col.column_name}] [text] {nullable}'
+    if col.type_name == "CHAR":
+        return f"   [{col.column_name}] [varchar]({col.length}) {nullable}"
+    if col.type_name == "HEX":
+        return f"   [{col.column_name}] [binary]({col.length}) {nullable}"
+    if col.type_name == "INTEGER":
+        return f"   [{col.column_name}] [int] {nullable}"
+    if col.type_name == "DATE":
+        return f"   [{col.column_name}] [datetime] {nullable}"
+    if col.type_name == "NUMERIC":
+        return f"   [{col.column_name}] [decimal]({col.length},{col.scale}) {nullable}"
+    if col.type_name == "BIT":
+        return f"   [{col.column_name}] [boolean] {nullable}"
+    if col.type_name == "MEMO":
+        return f"   [{col.column_name}] [text] {nullable}"
     return ", ".join(list(map(str, col)))
 
 
 def convert_desc2(col):
     return ", ".join(list(map(str, col)))
+
+
 # table_name = [x[2] for x in crsr.tables(tableType='VIEW')]
 # open("views.txt", "w").write("\n".join(table_name))
 
@@ -47,7 +49,7 @@ def tables_cols():
             crsr.cancel()
         except pyodbc.Error as e:
             print(e)
-            if t != '':
+            if t != "":
                 res[t] = []
             crsr = connect()
 
@@ -63,11 +65,14 @@ def pkeys():
             if res.get(t) is None:
                 res[t] = []
             res[t].append(
-                f"   CONSTRAINT [{t}$0] PRIMARY KEY CLUSTERED ([" + "], [".join([c.column_name for c in cols]) + "])")
+                f"   CONSTRAINT [{t}$0] PRIMARY KEY CLUSTERED (["
+                + "], [".join([c.column_name for c in cols])
+                + "])"
+            )
             crsr.cancel()
         except pyodbc.Error as e:
             print(e)
-            if t != '':
+            if t != "":
                 res[t] = []
             crsr = connect()
     json.dump(res, open("pkeys.json", "w"), indent=2)
@@ -85,7 +90,7 @@ def fkeys():
             crsr.cancel()
         except pyodbc.Error as e:
             print(e)
-            if t != '':
+            if t != "":
                 res[t] = []
             crsr = connect()
 
@@ -98,7 +103,7 @@ def tables_create():
             wh.write(")\n\nGO\n")
 
 
-if __name__ == '__main__':
+def schema():
     tables_cols()
     pkeys()
     # fkeys()

db.bat

@@ -0,0 +1,3 @@
+cd /d %~dp0
+pyinstaller -F --path %~dp0 db.py
+pause

db.py

@@ -0,0 +1,25 @@
+import config
+import database
+import plac
+
+
+class DatabaseTools:
+    commands = ["create", "run", "schema"]
+
+    def __init__(self):
+        self.cfg = config.Config()
+
+    def create(self, config_file: str):
+        config_file = self.cfg.system_dir + f"\\SQL\\config\\{config_file}.json"
+        database.create(config_file)
+
+    def run(self):
+        batch_dir = self.cfg.system_dir + "\\SQL\\batch"
+        database.run(batch_dir)
+
+    def schema(self):
+        database.schema()
+
+
+if __name__ == "__main__":
+    plac.Interpreter.call(DatabaseTools)

db.spec

@@ -0,0 +1,40 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+
+block_cipher = None
+
+
+a = Analysis(['db.py'],
+             pathex=['C:\\Projekte\\tools\\'],
+             binaries=[],
+             datas=[],
+             hiddenimports=[],
+             hookspath=[],
+             hooksconfig={},
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher,
+             noarchive=False)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,  
+          [],
+          name='db',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          upx_exclude=[],
+          runtime_tmpdir=None,
+          console=True,
+          disable_windowed_traceback=False,
+          target_arch=None,
+          codesign_identity=None,
+          entitlements_file=None )

requirements.txt

@@ -1,5 +1,7 @@
 Flask>=2.2.0
 beautifulsoup4
+numpy
 pandas
-sqlalchemy
-zipfile
+plac
+pyodbc
+sqlalchemy

tox.ini

@@ -1,5 +1,5 @@
 [flake8]
-ignore = E712, W504
+ignore = E712, W503, W504
 max-line-length = 140
 # exclude = tests/*
 # max-complexity = 10