- gcstruct Test ohne KTR

- db_create.py mit ausgelagerter Klasse für inspect

dbtools/OPTIMA.csv

@@ -72,25 +72,25 @@ source;target;filter;query;iterative
 [{0}].[dbo].[VEH_TRANSACT_GRID];VEH_TRANSACT_GRID;;;
 [{0}].[dbo].[VEH_TRANSACT_GROUPS];VEH_TRANSACT_GROUPS;;;
 [{0}].[dbo].[VEH_TRANSMISSION_CODES];VEH_TRANSMISSION_CODES;;;
-[{0}].[dbo].[VPP25];VPP25;;;
-[{0}].[dbo].[VPP2C];VPP2C;;;
-[{0}].[dbo].[VPP43];VPP43;;;
-[{0}].[dbo].[VPP48];VPP48;;;
-[{0}].[dbo].[VPP4K];VPP4K;;;
-[{0}].[dbo].[VPP51];VPP51;;;
-[{0}].[dbo].[VPP53];VPP53;;;
-[{0}].[dbo].[VPP5A];VPP5A;;;
-[{0}].[dbo].[VPP5M];VPP5M;;;
-[{0}].[dbo].[VPP5Q];VPP5Q;;;
-[{0}].[dbo].[VPP5R];VPP5R;;;
-[{0}].[dbo].[VPP61];VPP61;;;
-[{0}].[dbo].[VPP65];VPP65;;;
-[{0}].[dbo].[VPP72];VPP72;;;
-[{0}].[dbo].[VPP73];VPP73;;;
-[{0}].[dbo].[VPP74];VPP74;;;
-[{0}].[dbo].[VPP91];VPP91;;;
-[{0}].[dbo].[VPP93];VPP93;;;
-[{0}].[dbo].[VPPT1];VPPT1;;;
+[{0}].[dbo].[vPP25];VPP25;;;
+[{0}].[dbo].[vPP2C];VPP2C;;;
+[{0}].[dbo].[vPP43];VPP43;;;
+[{0}].[dbo].[vPP48];VPP48;;;
+[{0}].[dbo].[vPP4K];VPP4K;;;
+[{0}].[dbo].[vPP51];VPP51;;;
+[{0}].[dbo].[vPP53];VPP53;;;
+[{0}].[dbo].[vPP5A];VPP5A;;;
+[{0}].[dbo].[vPP5M];VPP5M;;;
+[{0}].[dbo].[vPP5Q];VPP5Q;;;
+[{0}].[dbo].[vPP5R];VPP5R;;;
+[{0}].[dbo].[vPP61];VPP61;;;
+[{0}].[dbo].[vPP65];VPP65;;;
+[{0}].[dbo].[vPP72];VPP72;;;
+[{0}].[dbo].[vPP73];VPP73;;;
+[{0}].[dbo].[vPP74];VPP74;;;
+[{0}].[dbo].[vPP91];VPP91;;;
+[{0}].[dbo].[vPP93];VPP93;;;
+[{0}].[dbo].[vPPT1];VPPT1;;;
 [{0}].[dbo].[WORKING_PERIOD];WORKING_PERIOD;;;
 [{0}].[dbo].[WORKSHOP_ORDER_SPLIT];WORKSHOP_ORDER_SPLIT;;;
 [{0}].[dbo].[WORKSHOP_PLANNING];WORKSHOP_PLANNING;;;

dbtools/OPTIMA.json

@@ -0,0 +1,9 @@
+{
+    "csv_file": "OPTIMA.csv",
+    "clients": {"desk01": "desk01"},
+    "filter": ["01.01.2018", "01.01.2019"],
+    "source_dsn": {"user": "sa", "pass": "Mffu3011#", "server": "192.168.2.41\\GLOBALCUBE", "database": "desk01", "driver": "mssql", "schema": "dbo"},
+    "target_dsn": {"user": "sa", "pass": "Mffu3011#", "server": "192.168.2.41\\GLOBALCUBE", "database": "OPTIMA", "driver": "mssql", "schema": "import"},
+    "stage_dir": "C:\\GlobalCube\\System\\OPTIMA\\SQL\\temp",
+    "batch_dir": "C:\\GlobalCube\\System\\OPTIMA\\SQL\\batch"
+}

dbtools/db_create.py

@@ -19,8 +19,40 @@ cfg = DbCreateConfig(**{
 })
 
 
+class database_inspect():
+    def __init__(self, dsn):
+        self.dsn = dsn
+        self.engine = create_engine(conn_string(self.dsn))
+        self.insp = inspect(self.engine)
+
+    def get_tables(self):
+        self.tables = self.insp.get_table_names(schema=self.dsn['schema']) + self.insp.get_view_names(schema=self.dsn['schema'])
+        return self.tables
+
+    def get_prefix(self):
+        source_tables_prefix = dict(enumerate(sorted(list(set([t.split('$')[0] for t in self.tables if '$' in t]))), 1))
+        if len(source_tables_prefix) == 0:
+            q = self.engine.execute('select name FROM sys.databases')
+            source_tables_prefix = [x[0] for x in q.fetchall()]
+        return source_tables_prefix
+
+    def get_columns(self, table):
+        source_insp_cols = self.insp.get_columns(table)
+        if len(source_insp_cols) == 0:
+            q = self.engine.execute(f"SELECT COLUMN_NAME as name FROM information_schema.columns WHERE TABLE_NAME = '{self.convert_table(table)}'")
+            source_insp_cols = q.fetchall()
+        return set([col['name'] for col in source_insp_cols])
+
+    def covert_table(self, table):
+        if '.' in table:
+            table = table.split('.')[-1]
+        if '[' in table:
+            table = table[1:-1]
+        return table
+
+
 @plac.pos('config_file', '', type=str)
-def create(config_file='CARLO.json'):
+def create(config_file='dbtools/OPTIMA.json'):
     cfg_import = json.load(open(config_file, 'r', encoding='ansi'))
     base_dir = Path(config_file).resolve().parent
     cfg_import['name'] = Path(config_file).stem
@@ -30,44 +62,46 @@ def create(config_file='CARLO.json'):
         cfg_import['batch_dir'] = str(base_dir.joinpath(cfg_import['batch_dir']).resolve())
     cfg = DbCreateConfig(**cfg_import)
 
-    df = pd.read_csv(str(base_dir) + "\\" + cfg.csv_file, sep=';', encoding='ansi')
+    df = pd.read_csv(f"{base_dir}/{cfg.csv_file}", sep=';', encoding='ansi')
     config = df[df['target'].notnull()]
     print(config.head())
 
-    source_db = create_engine(conn_string(cfg.source_dsn))
-    source_insp = inspect(source_db)
-    source_tables = source_insp.get_table_names(schema=cfg.source_dsn['schema'])
-    source_tables_prefix = dict(enumerate(sorted(list(set([t.split('$')[0] for t in source_tables if '$' in t]))), 1))
-    print(source_tables_prefix)
+    source_db = database_inspect(cfg.source_dsn)
+    source_tables = source_db.get_tables()
+    print(source_db.get_prefix())
 
-    target_db = create_engine(conn_string(cfg.target_dsn))
-    target_insp = inspect(target_db)
-    target_tables = target_insp.get_table_names(schema=cfg.target_dsn['schema'])
+    target_db = database_inspect(cfg.target_dsn)
+    target_tables = target_db.get_tables()
 
     for index, current_table in config.iterrows():
-        with open(cfg.batch_dir + "\\" + current_table['target'] + '.bat', 'w', encoding='cp850') as f:
+        with open(f"{cfg.batch_dir}/{current_table['target']}.bat", 'w', encoding='cp850') as f:
             f.write('@echo off \n')
             f.write('rem ==' + current_table['target'] + '==\n')
 
             if not current_table['target'] in target_tables:
                 f.write(f"echo Ziel-Tabelle '{current_table['target']}' existiert nicht!\n")
+                print(f"Ziel-Tabelle '{current_table['target']}' existiert nicht!")
                 continue
 
             f.write(f"del {cfg.stage_dir}\\{current_table['target']}*.* /Q /F >nul 2>nul \n")
             f.write(f"sqlcmd.exe {bcp_conn_params(cfg.target_dsn)} -p -Q \"TRUNCATE TABLE [{cfg.target_dsn['schema']}].[{current_table['target']}]\" \n")
 
-            target_insp_cols = target_insp.get_columns(current_table['target'], schema=cfg.target_dsn['schema'])
-            target_columns_list = [col['name'] for col in target_insp_cols]
+            target_columns_list = target_db.get_columns(current_table['target'])
+            if 'CLIENT_DB' in target_columns_list:
+                target_columns_list.remove('CLIENT_DB')
+                target_columns_list.append('Client_DB')
             target_columns = set(target_columns_list)
 
             for client_db, prefix in cfg.clients.items():
                 source_table = current_table['source'].format(prefix)
                 if source_table not in source_tables:
-                    f.write(f"echo Quell-Tabelle '{source_table}' existiert nicht!\n")
-                    continue
+                    source_table2 = source_db.convert_table(source_table)
+                    if source_table2 not in source_tables:
+                        f.write(f"echo Quell-Tabelle '{source_table}' existiert nicht!\n")
+                        print(f"Quell-Tabelle '{source_table}' existiert nicht!")
+                        continue
 
-                source_insp_cols = source_insp.get_columns(source_table)
-                source_columns = set([col['name'] for col in source_insp_cols])
+                source_columns = source_db.get_columns(source_table)
 
                 if not pd.isnull(current_table['query']):
                     select_query = current_table['query'].format(prefix, cfg.filter[0], cfg.filter[1])
@@ -87,6 +121,7 @@ def create(config_file='CARLO.json'):
                 diff2 = target_columns.difference(source_columns)
                 if 'Client_DB' not in diff2:
                     f.write("echo Spalte 'Client_DB' fehlt!\n")
+                    print(f"Ziel-Tabelle '{current_table['target']}' Spalte 'Client_DB' fehlt!")
                     continue
                 diff2.remove('Client_DB')
                 if len(diff2) > 0:
@@ -96,7 +131,7 @@ def create(config_file='CARLO.json'):
                 select_columns = ''
                 for col in target_columns_list:
                     if col in intersect:
-                        select_columns += "T1.[" + col + "], "
+                        select_columns += f"T1.[{col}], "
                     elif col == 'Client_DB':
                         select_columns += "'" + client_db + "' as \\\"Client_DB\\\", "
                     else:
@@ -114,13 +149,13 @@ def create(config_file='CARLO.json'):
                 f.write(f"bcp [{cfg.target_dsn['schema']}].[{current_table['target']}] in \"{stage_csv}\" {bcp_conn_params(cfg.target_dsn)} -c -C 65001 -e \"{stage_csv[:-4]}.in.log\" > \"{stage_csv[:-4]}.bcp2.log\" \n")
                 f.write(f"type \"{stage_csv[:-4]}.bcp2.log\" | findstr -v \"1000\" \n")
 
-    with open(cfg.batch_dir + "\\_" + cfg.name + ".bat", "w", encoding="cp850") as f:
+    with open(f"{cfg.batch_dir}/_{cfg.name}.bat", 'w', encoding='cp850') as f:
         f.write("@echo off & cd /d %~dp0 \n")
         f.write(f"del {cfg.stage_dir}\\*.* /Q /F >nul 2>nul \n\n")
         for index, current_table in config.iterrows():
-            f.write("echo ==" + current_table['target'] + "==\n")
-            f.write("echo " + current_table['target'] + " >CON \n")
-            f.write("call " + current_table['target'] + ".bat\n\n")
+            f.write(f"echo =={current_table['target']}==\n")
+            f.write(f"echo {current_table['target']} >CON \n")
+            f.write(f"call {current_table['target']}.bat\n\n")
 
 
 if __name__ == '__main__':

gcstruct.py

@@ -290,10 +290,10 @@ class GCStruct():
             if t_to == 'SKR51':
                 df_source['SKR51'] = df_source['Konto_Nr']
                 # print(df_translate[t_to].info())
-            elif t_to == 'KTR':
-                df_source['KTR'] = 'KTR_SC_' + df_source['Marke'] + '_' + df_source['Kostenträger']
-                df_source['KTR'] = np.where(df_source['Ebene1_KST'] == '1-NW', 'KTR_NW_' + df_source['Marke'] + '_' + df_source['Kostenträger'], df_source[t_to])
-                df_source['KTR'] = np.where(df_source['Ebene1_KST'] == '3-TZ', 'KTR_TZ_' + df_source['Kostenträger'], df_source[t_to])
+            # elif t_to == 'KTR':
+            #     df_source['KTR'] = 'KTR_SC_' + df_source['Marke'] + '_' + df_source['Kostenträger']
+            #     df_source['KTR'] = np.where(df_source['Ebene1_KST'] == '1-NW', 'KTR_NW_' + df_source['Marke'] + '_' + df_source['Kostenträger'], df_source[t_to])
+            #     df_source['KTR'] = np.where(df_source['Ebene1_KST'] == '3-TZ', 'KTR_TZ_' + df_source['Kostenträger'], df_source[t_to])
             else:
                 df_source[t_to] = df_source[t_from].apply(lambda x: t_to + '_' + x)
             df_source = df_source.merge(df_translate[t_to], how='left', on=[t_to], suffixes=(None, '_' + t_to))