- db_create.py für Autoline und Optima

- cet als exe

dbtools/AUTOLINE.csv

@@ -0,0 +1,28 @@
+source;target;filter;query;iterative
+DD_{0}_CAPVehiclePrices;DD_CAPVehiclePrices;;;
+DD_{0}_ModelRecords;DD_ModelRecords;;;
+GB_{0}_UserDetails;GB_UserDetails;;;
+MK_{0}_CompanyRecords;MK_CompanyRecords;;;
+MK_{0}_CustomerRecords;MK_CustomerRecords;;;
+MK_{0}_DemonstrationVehicles;MK_DemonstrationVehicles;;;
+MK_{0}_DemoVehicleTransactions;MK_DemoVehicleTransactions;;;
+MK_{0}_Teams;MK_Teams;;;
+MK_{0}_TransactionTypes;MK_TransactionTypes;;;
+MK_{0}_TypesOfBusiness;MK_TypesOfBusiness;;;
+MK_{0}_VehicleRecords;MK_VehicleRecords;;;
+MK_{0}_VehicleStatuses;MK_VehicleStatuses;;;
+MK_{0}_VehicleStockRecord;MK_VehicleStockRecord;;;
+SL_{0}_InvoiceCreditNoteCF;SL_InvoiceCreditNoteCF;;;
+SL_{0}_SalesLedgerAccounts;SL_SalesLedgerAccounts;;;
+SL_{0}_SLTransactions;SL_SLTransactions;;;
+SO_{0}_HeaderRecords;SO_HeaderRecords;;;
+SO_{0}_LabourLines;SO_LabourLines;;;
+SO_{0}_LabourRates;SO_LabourRates;;;
+SO_{0}_PartsLines;SO_PartsLines;;;
+VS_{0}_NewVehicleStock;VS_NewVehicleStock;;;
+VS_{0}_OrderStatuses;VS_OrderStatuses;;;
+VS_{0}_PreviousUsage;VS_PreviousUsage;;;
+VS_{0}_PurchaseOrderTypes;VS_PurchaseOrderTypes;;;
+VS_{0}_SalesOrderTypes;VS_SalesOrderTypes;;;
+VS_{0}_UsedVehicleStock;VS_UsedVehicleStock;;;
+VS_{0}_VehicleSpecification;VS_VehicleSpecification;;;

dbtools/AUTOLINE.json

@@ -0,0 +1,9 @@
+{
+    "csv_file": "AUTOLINE.csv",
+    "clients": {"00": "00", "01": "01", "10": "10", "12": "12"},
+    "filter": ["01.01.2018", "01.01.2019"],
+    "source_dsn": {"user": "kcc", "password": "kcc123", "server": "Autoline_direkt64", "database": "", "driver": "kisam", "schema": ""},
+    "target_dsn": {"user": "sa", "password": "Mffu3011#", "server": "localhost\\GLOBALCUBE", "database": "AUTOLINE", "driver": "mssql", "schema": "import"},
+    "stage_dir": "..\\temp",
+    "batch_dir": "..\\batch"
+}

dbtools/CARLO.json

@@ -2,8 +2,8 @@
     "csv_file": "CARLO.csv",
     "clients": {"1": "M und S Fahrzeughandel GmbH"},
     "filter": ["01.01.2018", "01.01.2019"],
-    "source_dsn": {"user": "sa", "pass": "Mffu3011#", "server": "192.168.2.41\\GLOBALCUBE", "database": "DE0017", "driver": "mssql", "schema": "dbo"},
-    "target_dsn": {"user": "sa", "pass": "Mffu3011#", "server": "192.168.2.41\\GLOBALCUBE", "database": "CARLO2", "driver": "mssql", "schema": "import"},
+    "source_dsn": {"user": "sa", "password": "Mffu3011#", "server": "192.168.2.41\\GLOBALCUBE", "database": "DE0017", "driver": "mssql", "schema": "dbo"},
+    "target_dsn": {"user": "sa", "password": "Mffu3011#", "server": "192.168.2.41\\GLOBALCUBE", "database": "CARLO2", "driver": "mssql", "schema": "import"},
     "stage_dir": "C:\\GlobalCube\\System\\CARLO\\SQL\\temp",
     "batch_dir": "C:\\GlobalCube\\System\\CARLO\\SQL\\batch"
 }

dbtools/OPTIMA.json

@@ -2,8 +2,8 @@
     "csv_file": "OPTIMA.csv",
     "clients": {"desk01": "desk01"},
     "filter": ["01.01.2018", "01.01.2019"],
-    "source_dsn": {"user": "sa", "pass": "Mffu3011#", "server": "192.168.2.41\\GLOBALCUBE", "database": "desk01", "driver": "mssql", "schema": "dbo"},
-    "target_dsn": {"user": "sa", "pass": "Mffu3011#", "server": "192.168.2.41\\GLOBALCUBE", "database": "OPTIMA", "driver": "mssql", "schema": "import"},
+    "source_dsn": {"user": "sa", "password": "Mffu3011#", "server": "192.168.2.41\\GLOBALCUBE", "database": "desk01", "driver": "mssql", "schema": "dbo"},
+    "target_dsn": {"user": "sa", "password": "Mffu3011#", "server": "192.168.2.41\\GLOBALCUBE", "database": "OPTIMA", "driver": "mssql", "schema": "import"},
     "stage_dir": "C:\\GlobalCube\\System\\OPTIMA\\SQL\\temp",
     "batch_dir": "C:\\GlobalCube\\System\\OPTIMA\\SQL\\batch"
 }

dbtools/cet.py

@@ -7,13 +7,15 @@ import pyodbc
 @plac.pos('csv_file', '', type=str)
 @plac.opt('Server', 'Hostname or DSN', type=str)
 @plac.opt('database', '', type=str)
-@plac.opt('user', '', type=str)
+@plac.opt('User', '', type=str)
 @plac.opt('Password', '', type=str)
 @plac.flg('charset', '')
 @plac.opt('Codepage', '', type=str)
 @plac.opt('errorlog', '', type=str)
-def run(query, mode, csv_file, Server='localhost\\GLOBALCUBE', database='master', user='sa', Password='Mffu3011#', charset=False, Codepage='cp65001', errorlog='error.log'):
-    dsn = f"dsn={Server};uid={user};pwd={Password}"
+def run(query, mode, csv_file, Server='localhost\\GLOBALCUBE', database='master', User='sa', Password='Mffu3011#', charset=False, Codepage='65001', errorlog='error.log'):
+    dsn = f"dsn={Server};uid={User};pwd={Password}"
+    if Codepage.isnumeric():
+        Codepage = 'cp' + Codepage
     if mode == 'queryout':
         queryout(dsn, query, csv_file, Codepage, errorlog)
         return
@@ -25,9 +27,9 @@ def queryout(dsn, query, csv_file, codepage, errorlog):
         conn = pyodbc.connect(dsn)
         cursor = conn.cursor()
         cursor.execute(query)
-        with open(csv_file, 'w', encoding=codepage, newline='\r\n') as fwh:
+        with open(csv_file, 'w', encoding=codepage) as fwh:
             while row := cursor.fetchone():
-                fwh.write('\t'.join(map(str, row)))
+                fwh.write('\t'.join(map(str, row)).replace('None', '').replace('False', '0').replace('True', '1') + '\n')
     except pyodbc.InterfaceError as e:
         print(e.args[1])
 

dbtools/cet.spec

@@ -0,0 +1,33 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+block_cipher = None
+
+
+a = Analysis(['cet.py'],
+             pathex=['C:\\Projekte\\Python\\dbtools'],
+             binaries=[],
+             datas=[],
+             hiddenimports=[],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher,
+             noarchive=False)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          [],
+          name='cet',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          upx_exclude=[],
+          runtime_tmpdir=None,
+          console=True )

dbtools/database.py

@@ -3,6 +3,7 @@ def conn_string(dsn):
         return f"mssql+pyodbc://{dsn['user']}:{dsn['pass']}@{dsn['server']}/{dsn['database']}?driver=SQL+Server+Native+Client+11.0"
     if dsn['driver'] == 'mysql':
         return f"mysql+pymysql://{dsn['user']}:{dsn['pass']}@{dsn['server']}/{dsn['database']}?charset=utf8mb4"
+    return f"pyodbc://{dsn['user']}:{dsn['pass']}@{dsn['server']}/{dsn['database']}?driver={dsn['driver']}"
 
 
 def bcp_conn_params(dsn):

dbtools/db_create.py

@@ -1,49 +1,70 @@
-import plac
-import pandas as pd
-from sqlalchemy import create_engine, inspect
-from database import bcp_conn_params, conn_string
 import json
-from pathlib import Path
 from collections import namedtuple
+from pathlib import Path
+
+import pandas as pd
+import plac
+import pyodbc
+
 DbCreateConfig = namedtuple('DbCreateConfig', 'name csv_file clients filter source_dsn target_dsn stage_dir batch_dir')
+DsnConfig = namedtuple('DsnConfig', 'user password server database driver schema')
 
 cfg = DbCreateConfig(**{
     'name': 'CARLO',
     'csv_file': 'CARLO.csv',
     'clients': {'1': 'M und S Fahrzeughandel GmbH'},
     'filter': ['01.01.2018', '01.01.2019'],
-    'source_dsn': {'user': 'sa', 'pass': 'Mffu3011#', 'server': 'GC-SERVER1\\GLOBALCUBE', 'database': 'DE0017', 'driver': 'mssql', 'schema': 'dbo'},
-    'target_dsn': {'user': 'sa', 'pass': 'Mffu3011#', 'server': 'GC-SERVER1\\GLOBALCUBE', 'database': 'CARLO2', 'driver': 'mssql', 'schema': 'import'},
+    'source_dsn': {'user': 'sa', 'password': 'Mffu3011#', 'server': 'GC-SERVER1\\GLOBALCUBE', 'database': 'DE0017', 'driver': 'mssql', 'schema': 'dbo'},
+    'target_dsn': {'user': 'sa', 'password': 'Mffu3011#', 'server': 'GC-SERVER1\\GLOBALCUBE', 'database': 'CARLO2', 'driver': 'mssql', 'schema': 'import'},
     'stage_dir': '..\\temp',
     'batch_dir': '..\\batch'
 })
 
 
 class database_inspect():
+    tables = []
+
     def __init__(self, dsn):
-        self.dsn = dsn
-        self.engine = create_engine(conn_string(self.dsn))
-        self.insp = inspect(self.engine)
+        self.dsn = DsnConfig(**dsn)
+        self.cursor = self.connect()
+
+    def conn_string(self):
+        if self.dsn.driver == 'mssql':
+            return 'Driver={SQL Server Native Client 11.0};' + f"Server={self.dsn.server};Database={self.dsn.database};Uid={self.dsn.user};Pwd={self.dsn.password}"
+        if self.dsn.driver == 'mysql':
+            return f"mysql+pymysql://{self.dsn.user}:{self.dsn.password}@{self.dsn.server}/{self.dsn.database}?charset=utf8mb4"
+        return f"DSN={self.dsn.server};UID={self.dsn.user};PWD={self.dsn.password}"
+
+    def bcp_conn_params(self):
+        return f"-S {self.dsn.server} -d {self.dsn.database} -U {self.dsn.user} -P {self.dsn.password}"
+
+    def connect(self):
+        c = pyodbc.connect(self.conn_string())
+        return c.cursor()
 
     def get_tables(self):
-        self.tables = self.insp.get_table_names(schema=self.dsn['schema']) + self.insp.get_view_names(schema=self.dsn['schema'])
+        tables = [x[2] for x in self.cursor.tables(tableType='TABLE')]
+        views = [x[2] for x in self.cursor.tables(tableType='VIEW')]
+        self.tables = tables + views
         return self.tables
 
     def get_prefix(self):
+        if (len(self.tables)) == 0:
+            self.get_tables()
         source_tables_prefix = dict(enumerate(sorted(list(set([t.split('$')[0] for t in self.tables if '$' in t]))), 1))
         if len(source_tables_prefix) == 0:
-            q = self.engine.execute('select name FROM sys.databases')
+            q = self.cursor.execute('select name FROM sys.databases')
             source_tables_prefix = [x[0] for x in q.fetchall()]
         return source_tables_prefix
 
     def get_columns(self, table):
-        source_insp_cols = self.insp.get_columns(table)
+        source_insp_cols = [col.column_name for col in self.cursor.columns(table=table)]
         if len(source_insp_cols) == 0:
-            q = self.engine.execute(f"SELECT COLUMN_NAME as name FROM information_schema.columns WHERE TABLE_NAME = '{self.convert_table(table)}'")
-            source_insp_cols = q.fetchall()
-        return set([col['name'] for col in source_insp_cols])
+            q = self.cursor.execute(f"SELECT COLUMN_NAME as column_name FROM information_schema.columns WHERE TABLE_NAME = '{self.convert_table(table)}'")
+            source_insp_cols = [col[0] for col in q.fetchall()]
+        return source_insp_cols
 
-    def covert_table(self, table):
+    def convert_table(self, table):
         if '.' in table:
             table = table.split('.')[-1]
         if '[' in table:
@@ -84,7 +105,7 @@ def create(config_file='dbtools/OPTIMA.json'):
                 continue
 
             f.write(f"del {cfg.stage_dir}\\{current_table['target']}*.* /Q /F >nul 2>nul \n")
-            f.write(f"sqlcmd.exe {bcp_conn_params(cfg.target_dsn)} -p -Q \"TRUNCATE TABLE [{cfg.target_dsn['schema']}].[{current_table['target']}]\" \n")
+            f.write(f"sqlcmd.exe {target_db.bcp_conn_params()} -p -Q \"TRUNCATE TABLE [{cfg.target_dsn['schema']}].[{current_table['target']}]\" \n")
 
             target_columns_list = target_db.get_columns(current_table['target'])
             if 'CLIENT_DB' in target_columns_list:
@@ -101,17 +122,7 @@ def create(config_file='dbtools/OPTIMA.json'):
                         print(f"Quell-Tabelle '{source_table}' existiert nicht!")
                         continue
 
-                source_columns = source_db.get_columns(source_table)
-
-                if not pd.isnull(current_table['query']):
-                    select_query = current_table['query'].format(prefix, cfg.filter[0], cfg.filter[1])
-                elif '.' in source_table:
-                    select_query = f"SELECT T1.* FROM {source_table} T1 "
-                else:
-                    select_query = f"SELECT T1.* FROM [{cfg.source_dsn['schema']}].[{source_table}] T1 "
-
-                if not pd.isnull(current_table['filter']):
-                    select_query += " WHERE " + current_table['filter'].format("", cfg.filter[0], cfg.filter[1])
+                source_columns = set(source_db.get_columns(source_table))
 
                 intersect = source_columns.intersection(target_columns)
                 # print("Auf beiden Seiten: " + ";".join(intersect))
@@ -127,11 +138,20 @@ def create(config_file='dbtools/OPTIMA.json'):
                 if len(diff2) > 0:
                     f.write("rem Nur in Ziel:   " + ";".join(diff2) + "\n")
 
+                if not pd.isnull(current_table['query']):
+                    select_query = current_table['query'].format(prefix, cfg.filter[0], cfg.filter[1])
+                elif '.' in source_table or cfg.source_dsn['schema'] == '':
+                    select_query = f"SELECT T1.* FROM \\\"{source_table}\\\" T1 "
+                else:
+                    select_query = f"SELECT T1.* FROM [{cfg.source_dsn['schema']}].[{source_table}] T1 "
+
+                if not pd.isnull(current_table['filter']):
+                    select_query += " WHERE " + current_table['filter'].format("", cfg.filter[0], cfg.filter[1])
                 # select_columns = "T1.[" + "], T1.[".join(intersect) + "],"
                 select_columns = ''
                 for col in target_columns_list:
                     if col in intersect:
-                        select_columns += f"T1.[{col}], "
+                        select_columns += f"T1.\\\"{col}\\\", "
                     elif col == 'Client_DB':
                         select_columns += "'" + client_db + "' as \\\"Client_DB\\\", "
                     else:
@@ -144,9 +164,10 @@ def create(config_file='dbtools/OPTIMA.json'):
                 # insert_query = f"LOAD DATA INFILE '{stage_csv}' INTO TABLE {current_table['target']} FIELDS TERMINATED BY ',' ENCLOSED BY '\"' LINES TERMINATED BY '\n';"
 
                 # print(select_query)
-                f.write(f"bcp \"{select_query}\" queryout \"{stage_csv}\" {bcp_conn_params(cfg.source_dsn)} -c -C 65001 -e \"{stage_csv[:-4]}.queryout.log\" > \"{stage_csv[:-4]}.bcp1.log\" \n")
+                bulk_copy = 'bcp' if cfg.source_dsn['driver'] == 'mssql' else 'cet'
+                f.write(f"{bulk_copy} \"{select_query}\" queryout \"{stage_csv}\" {source_db.bcp_conn_params()} -c -C 65001 -e \"{stage_csv[:-4]}.queryout.log\" > \"{stage_csv[:-4]}.bcp1.log\" \n")
                 f.write(f"type \"{stage_csv[:-4]}.bcp1.log\" | findstr -v \"1000\" \n")
-                f.write(f"bcp [{cfg.target_dsn['schema']}].[{current_table['target']}] in \"{stage_csv}\" {bcp_conn_params(cfg.target_dsn)} -c -C 65001 -e \"{stage_csv[:-4]}.in.log\" > \"{stage_csv[:-4]}.bcp2.log\" \n")
+                f.write(f"bcp [{cfg.target_dsn['schema']}].[{current_table['target']}] in \"{stage_csv}\" {target_db.bcp_conn_params()} -c -C 65001 -e \"{stage_csv[:-4]}.in.log\" > \"{stage_csv[:-4]}.bcp2.log\" \n")
                 f.write(f"type \"{stage_csv[:-4]}.bcp2.log\" | findstr -v \"1000\" \n")
 
     with open(f"{cfg.batch_dir}/_{cfg.name}.bat", 'w', encoding='cp850') as f:

dbtools/db_schema.py

@@ -1,30 +1,105 @@
 import pyodbc
 import json
-c = pyodbc.connect("DSN=Autoline_direkt64;UID=kcc;PWD=kcc123")
-crsr = c.cursor()
 
-# table_name = [x[2] for x in crsr.tables(tableType='TABLE')]
-# open("tables.txt", "w").write("\n".join(table_name))
 
-with open("tables.txt", "r") as rh:
-    tables = rh.readlines()
+def connect():
+    c = pyodbc.connect("DSN=Autoline_direkt64;UID=kcc;PWD=kcc123")
+    return c.cursor()
+
+
+def convert_desc(col):
+    nullable = "NULL" if col.nullable == 1 else "NOT NULL"
+    if col.type_name == 'CHAR':
+        return f'   [{col.column_name}] [varchar]({col.length}) {nullable}'
+    if col.type_name == 'HEX':
+        return f'   [{col.column_name}] [binary]({col.length}) {nullable}'
+    if col.type_name == 'INTEGER':
+        return f'   [{col.column_name}] [int] {nullable}'
+    if col.type_name == 'DATE':
+        return f'   [{col.column_name}] [datetime] {nullable}'
+    if col.type_name == 'NUMERIC':
+        return f'   [{col.column_name}] [decimal]({col.length},{col.scale}) {nullable}'
+    if col.type_name == 'BIT':
+        return f'   [{col.column_name}] [boolean] {nullable}'
+    if col.type_name == 'MEMO':
+        return f'   [{col.column_name}] [text] {nullable}'
+    return ", ".join(list(map(str, col)))
+
 
+def convert_desc2(col):
+    return ", ".join(list(map(str, col)))
+# table_name = [x[2] for x in crsr.tables(tableType='VIEW')]
+# open("views.txt", "w").write("\n".join(table_name))
+
+
+with open("tables.txt", "r") as rh:
+    tables = rh.read().split("\n")
 res = {}
 
 
-def convert_desc(x):
-    x = list(map(str, x))
-    # x[1] = str(x[1])
-    return ", ".join(x)
+def tables_cols():
+    crsr = connect()
+    for t in tables:
+        try:
+            cols = crsr.columns(table=t)
+            # print([x[0] for x in crsr.description])
+            res[t] = [convert_desc(c) for c in cols]
+            crsr.cancel()
+        except pyodbc.Error as e:
+            print(e)
+            if t != '':
+                res[t] = []
+            crsr = connect()
+
+    json.dump(res, open("schema.json", "w"), indent=2)
+
+
+def pkeys():
+    crsr = connect()
+    for t in tables:
+        try:
+            cols = crsr.primaryKeys(table=t)
+            # print([x[0] for x in crsr.description])
+            if res.get(t) is None:
+                res[t] = []
+            res[t].append(
+                f"   CONSTRAINT [{t}$0] PRIMARY KEY CLUSTERED ([" + "], [".join([c.column_name for c in cols]) + "])")
+            crsr.cancel()
+        except pyodbc.Error as e:
+            print(e)
+            if t != '':
+                res[t] = []
+            crsr = connect()
+    json.dump(res, open("pkeys.json", "w"), indent=2)
+
+
+def fkeys():
+    crsr = connect()
+    for t in tables:
+        try:
+            cols = crsr.foreignKeys(table=t)
+            print([x[0] for x in crsr.description])
+            if res.get(t) is None:
+                res[t] = []
+            res[t].append([convert_desc2(c) for c in cols])
+            crsr.cancel()
+        except pyodbc.Error as e:
+            print(e)
+            if t != '':
+                res[t] = []
+            crsr = connect()
+
 
+def tables_create():
+    for t, cols in res.items():
+        with open("../sql_load/schema/AUTOLINE/tables/import." + t + ".sql", "w") as wh:
+            wh.write(f"CREATE TABLE [import].[{t}] (\n")
+            wh.write(",\n".join(cols))
+            wh.write(")\n\nGO\n")
 
-for t in tables:
-    try:
-        crsr.execute("SELECT * FROM " + t)
-        res[t[:-1]] = [convert_desc(x) for x in crsr.description]
-        crsr.cancel()
-    except pyodbc.Error:
-        res[t[:-1]] = []
 
-# print(res)
-json.dump(res, open("schema.json", "w"), indent=2)
+if __name__ == '__main__':
+    tables_cols()
+    pkeys()
+    # fkeys()
+    tables_create()