Finaler Stand IFT und GEBOS-Import BMW-Wien

tools/database/config/exec/GEBOS_Account.sql

@@ -0,0 +1,8 @@
+DELETE T1 FROM [dbo].[GEBOS_Account] T1 
+INNER JOIN [temp].[GEBOS_Account] T2 
+ON T1.[AccountNr] = T2.[AccountNr] 
+GO
+
+INSERT INTO [dbo].[GEBOS_Account]
+SELECT T1.* FROM [temp].[GEBOS_Account] T1
+GO

tools/database/config/exec/GEBOS_Activity.sql

@@ -0,0 +1,8 @@
+DELETE T1 FROM [dbo].[GEBOS_Activity] T1 
+INNER JOIN [temp].[GEBOS_Activity] T2 
+ON T1.[ActivityNr] = T2.[ActivityNr] 
+GO
+
+INSERT INTO [dbo].[GEBOS_Activity]
+SELECT T1.* FROM [temp].[GEBOS_Activity] T1
+GO

tools/database/config/exec/GEBOS_Activity_Type.sql

@@ -0,0 +1,8 @@
+DELETE T1 FROM [dbo].[GEBOS_Activity_Type] T1 
+INNER JOIN [temp].[GEBOS_Activity_Type] T2 
+ON T1.[ActivityTypeNr] = T2.[ActivityTypeNr] 
+GO
+
+INSERT INTO [dbo].[GEBOS_Activity_Type]
+SELECT T1.* FROM [temp].[GEBOS_Activity_Type] T1
+GO

tools/database/config/exec/GEBOS_Cost_Center.sql

@@ -0,0 +1,8 @@
+DELETE T1 FROM [dbo].[GEBOS_Cost_Center] T1 
+INNER JOIN [temp].[GEBOS_Cost_Center] T2 
+ON T1.[CostCenterNr] = T2.[CostCenterNr] 
+GO
+
+INSERT INTO [dbo].[GEBOS_Cost_Center]
+SELECT T1.* FROM [temp].[GEBOS_Cost_Center] T1
+GO

tools/database/config/exec/GEBOS_Department.sql

@@ -0,0 +1,8 @@
+DELETE T1 FROM [dbo].[GEBOS_Department] T1 
+INNER JOIN [temp].[GEBOS_Department] T2 
+ON T1.[DepartmentNr] = T2.[DepartmentNr] 
+GO
+
+INSERT INTO [dbo].[GEBOS_Department]
+SELECT T1.* FROM [temp].[GEBOS_Department] T1
+GO

tools/database/config/exec/GEBOS_Employee.sql

@@ -0,0 +1,8 @@
+DELETE T1 FROM [dbo].[GEBOS_Employee] T1 
+INNER JOIN [temp].[GEBOS_Employee] T2 
+ON T1.[PersNr] = T2.[PersNr] 
+GO
+
+INSERT INTO [dbo].[GEBOS_Employee]
+SELECT T1.* FROM [temp].[GEBOS_Employee] T1
+GO

tools/database/config/exec/GEBOS_Employee_Account.sql

@@ -0,0 +1,10 @@
+DELETE T1 FROM [dbo].[GEBOS_Employee_Account] T1 
+INNER JOIN [temp].[GEBOS_Employee_Account] T2 
+ON T1.[PersNr] = T2.[PersNr] 
+AND T1.[Day] = T2.[Day] 
+AND T1.[AccountNr] = T2.[AccountNr] 
+GO
+
+INSERT INTO [dbo].[GEBOS_Employee_Account]
+SELECT T1.* FROM [temp].[GEBOS_Employee_Account] T1
+GO

tools/database/config/exec/GEBOS_Employment_Type.sql

@@ -0,0 +1,8 @@
+DELETE T1 FROM [dbo].[GEBOS_Employment_Type] T1 
+INNER JOIN [temp].[GEBOS_Employment_Type] T2 
+ON T1.[EmploymentTypeNr] = T2.[EmploymentTypeNr] 
+GO
+
+INSERT INTO [dbo].[GEBOS_Employment_Type]
+SELECT T1.* FROM [temp].[GEBOS_Employment_Type] T1
+GO

tools/database/config/exec/GEBOS_Time_Model.sql

@@ -0,0 +1,8 @@
+DELETE T1 FROM [dbo].[GEBOS_Time_Model] T1 
+INNER JOIN [temp].[GEBOS_Time_Model] T2 
+ON T1.[TimeModelNr] = T2.[TimeModelNr] 
+GO
+
+INSERT INTO [dbo].[GEBOS_Time_Model]
+SELECT T1.* FROM [temp].[GEBOS_Time_Model] T1
+GO

tools/database/config/exec/GEBOS_Time_Tracking.sql

@@ -0,0 +1,11 @@
+DELETE T1 FROM [dbo].[GEBOS_Time_Tracking] T1 
+INNER JOIN [temp].[GEBOS_Time_Tracking] T2 
+ON T1.[PersNr] = T2.[PersNr] 
+AND T1.[Day] = T2.[Day] 
+AND T1.[ActivityTypeNr] = T2.[ActivityTypeNr] 
+AND T1.[StartingTime] = T2.[StartingTime] 
+GO
+
+INSERT INTO [dbo].[GEBOS_Time_Tracking]
+SELECT T1.*, datepart(HOUR, T1.[Duration]) * 60 + datepart(MINUTE, T1.[Duration]) as Zeitdauer FROM [temp].[GEBOS_Time_Tracking] T1
+GO

tools/database/config/exec/IFT_Actuals.sql

@@ -0,0 +1,10 @@
+DELETE T1 FROM [dbo].[IFT_Actuals] T1 
+INNER JOIN [temp].[IFT_Actuals] T2 
+ON T1.[Document Number] = T2.[Document Number] 
+AND T1.[Number of Line Item] = T2.[Number of Line Item]
+AND T1.[Posting Date] = T2.[Posting Date]
+GO
+
+INSERT INTO [dbo].[IFT_Actuals]
+SELECT T1.*, LEFT(T1.[Account Number GL], 7) FROM [temp].[IFT_Actuals] T1
+GO

tools/database/config/exec/IFT_Budget.sql

@@ -0,0 +1,6 @@
+TRUNCATE TABLE [dbo].[IFT_Budget]
+GO
+
+INSERT INTO [dbo].[IFT_Budget]
+SELECT T1.* FROM [temp].[IFT_Budget] T1
+GO

tools/database/config/exec/IFT_Commitment.sql

@@ -0,0 +1,6 @@
+TRUNCATE TABLE [dbo].[IFT_Commitment]
+GO
+
+INSERT INTO [dbo].[IFT_Commitment]
+SELECT T1.* FROM [temp].[IFT_Commitment] T1
+GO

tools/database/config/exec/IFT_Controlling_Documents.sql

@@ -0,0 +1,9 @@
+DELETE T1 FROM [dbo].[IFT_Controlling_Documents] T1  
+INNER JOIN [temp].[IFT_Controlling_Documents] T2 
+ON T1.[Document Number] = T2.[Document Number] 
+AND T1.[posting line] = T2.[posting line]
+GO
+
+INSERT INTO [dbo].[IFT_Controlling_Documents]
+SELECT T1.* FROM [temp].[IFT_Controlling_Documents] T1
+GO

tools/database/config/exec/IFT_Plan.sql

@@ -0,0 +1,6 @@
+TRUNCATE TABLE [dbo].[IFT_Plan]
+GO
+
+INSERT INTO [dbo].[IFT_Plan]
+SELECT T1.* FROM [temp].[IFT_Plan] T1
+GO

tools/database/gebos_backup.py

@@ -0,0 +1,27 @@
+import os
+from pathlib import Path
+
+
+def gebos_backup(base_dir=None):
+    if base_dir is None:
+        base_dir = 'E:\\GEBOS'
+
+    source_path = Path(base_dir) / 'data'
+    target_path = base_dir + '/archive'
+
+    for source_file in source_path.glob('*.csv'):
+        # print(source_file)
+        table, timestamp = source_file.name.split('2', 1)
+        timestamp = '2' + timestamp
+        year = timestamp[:4]
+        month = timestamp[4:6]
+        target = Path(f"{target_path}/{year}/{table}/{year}-{month}/{source_file.name}").absolute()
+        os.makedirs(target.parent, exist_ok=True)
+        if target.exists():
+            target.unlink()
+        print(target)
+        source_file.rename(target)
+
+
+if __name__ == '__main__':
+    gebos_backup()

tools/database/gebos_convert.py

@@ -1,28 +1,37 @@
-import csv
-from pathlib import Path
-
-
-def convert_dir(path):
-    source_path = Path(path)
-    target_path = source_path.parent.joinpath('staging')
-
-    for source_file in source_path.glob('*.csv'):
-        print(source_file.name)
-        target_file = target_path / source_file.name
-        convert_file(source_file, target_file)
-
-
-def convert_file(source, target):
-    with open(source, 'r', encoding='utf-8') as frh:
-        with open(target, 'w', encoding='latin-1') as fwh:
-            csv_reader = csv.reader(frh, delimiter=';')
-            csv_writer = csv.writer(fwh, delimiter='\t')
-            csv_writer.writerows(csv_reader)
-
-
-def main():
-    convert_dir('E:\\GEBOS\\data')
-
-
-if __name__ == '__main__':
-    main()
+import csv
+import re
+from pathlib import Path
+
+
+def convert_dir(path):
+    source_path = Path(path)
+    target_path = source_path.parent.joinpath('staging')
+
+    for source_file in source_path.glob('*.csv'):
+        print(source_file.name)
+        target_file = target_path / source_file.name
+        convert_file(source_file, target_file)
+
+
+def convert_file(source, target):
+    with open(source, 'r', encoding='utf-8', newline='') as frh:
+        with open(target, 'w', encoding='latin-1', newline='') as fwh:
+            csv_reader = csv.reader(frh, delimiter=';')
+            csv_writer = csv.writer(fwh, delimiter='\t')
+            for row in csv_reader:
+                converted = [convert_field(col) for col in row]
+                csv_writer.writerow(converted)
+
+
+def convert_field(col):
+    if re.match(r'\d:\d\d:\d\d', col):
+        col = '0' + col
+    return col
+
+
+def main():
+    convert_dir('E:\\GEBOS\\data')
+
+
+if __name__ == '__main__':
+    main()

tools/database/gebos_import.py

@@ -0,0 +1,52 @@
+from pathlib import Path
+import subprocess
+
+cfg = {
+    'schema': 'temp',
+    'scripts_dir': 'E:\\GlobalCube\\Tasks\\scripts',
+    'query_dir': 'E:\\GlobalCube\\System\\NAVISION\\SQL\\exec',
+}
+target_tables = {
+    'account2': 'GEBOS_Account',
+    'activity2': 'GEBOS_Activity',
+    'activitytype2': 'GEBOS_Activity_Type',
+    'costcenter2': 'GEBOS_Cost_Center',
+    'department2': 'GEBOS_Department',
+    'employee2': 'GEBOS_Employee',
+    'employeeaccount2': 'GEBOS_Employee_Account',
+    'employmenttype2': 'GEBOS_Employment_Type',
+    'timemodel2': 'GEBOS_Time_Model',
+    'timetracking2': 'GEBOS_Time_Tracking'
+}
+
+
+def task(args):
+    return subprocess.Popen(args, stdout=subprocess.DEVNULL).wait()
+
+
+def gebos_import(csv_file, target_table):
+    bulkcopy = str(Path(cfg['scripts_dir']) / 'bcp_import.bat')
+    sqlquery = str(Path(cfg['scripts_dir']) / 'sqlquery.bat')
+    sqlexec = str(Path(cfg['scripts_dir']) / 'sqlexec.bat')
+    query_file = target_table + '.sql'
+
+    task(f"{sqlquery} \"TRUNCATE TABLE [{cfg['schema']}].[{target_table}]\" ")
+    task(f"{bulkcopy} [{cfg['schema']}].[{target_table}] \"{csv_file}\" ")
+    task(f"{sqlexec} {query_file} \"{csv_file}\" ")
+
+
+def import_dir(base_dir=None):
+    if base_dir is None:
+        base_dir = 'E:\\GEBOS\\staging'
+
+    for csv_file in Path(base_dir).glob('*.csv'):
+        print(csv_file.name)
+        for key, table in target_tables.items():
+            if not csv_file.name.startswith(key):
+                continue
+            gebos_import(str(csv_file), table)
+            csv_file.unlink()
+
+
+if __name__ == '__main__':
+    import_dir()

tools/database/ift_backup.py

@@ -4,22 +4,24 @@ from pathlib import Path
 
 def ift_backup(base_dir=None):
     if base_dir is None:
-        base_dir = '/home/robert/projekte/python/BMW_Wien/IFT'
+        base_dir = 'E:\\IFT'
 
     source_path = Path(base_dir) / 'prod'
     target_path = base_dir + '/archive'
 
-    for source_file in source_path.glob('*.txt'):
+    for source_file in source_path.glob('*'):
         # print(source_file)
-        file_temp = source_file.name[:-4]
+        file_temp = source_file.name
         if file_temp.count('_') == 2:
             prefix, filetype, timestamp = file_temp.split('_')
             year = timestamp[:4]
             month = timestamp[4:6]
-            target = f"{target_path}/{filetype}/{year}/{year}-{month}/{source_file.name}"
-            os.makedirs(Path(target).parent, exist_ok=True)
+            target = Path(f"{target_path}/{year}/{filetype}/{year}-{month}/{source_file.name}").absolute()
+            os.makedirs(target.parent, exist_ok=True)
+            if target.exists():
+                target.unlink()
             print(target)
-            # source_file.rename(target)
+            source_file.rename(target)
 
 
 if __name__ == '__main__':

tools/database/ift_convert.py

@@ -1,4 +1,5 @@
 import csv
+import re
 from pathlib import Path
 from itertools import chain
 
@@ -24,6 +25,7 @@ class IFTConverter:
                 '010': self.import_config(base_dir + 'plan_line.txt')
             }
         }
+        self.is_number = re.compile(r'\d+\.\d+\-?$')
 
     def import_config(self, filename):
         with open(filename, 'r') as frh:
@@ -33,21 +35,26 @@ class IFTConverter:
 
     def convert_dir(self, path):
         source_path = Path(path)
-        target_path = source_path.parent.joinpath('prod_csv')
+        target_path = source_path.parent.joinpath('staging')
 
-        for filename in source_path.glob('*.txt'):
+        for filename in source_path.glob('*'):
             print(filename.name)
-            self.convert_file(filename, target_path)
+            if filename.name.count('_') < 2:
+                print('-> wrong file format')
+            elif filename.stat().st_size == 0:
+                print('-> file is empty!')
+            else: 
+                self.convert_file(filename, target_path)
 
     def convert_file(self, source, target_path):
         cfg = self.conversion_config(source.name)
-        target = target_path.joinpath(source.name[:-4] + '.csv')
+        target = target_path.joinpath(source.name + '.csv')
 
         content = {}
         last_key = list(cfg.keys())[-1]
 
-        with open(source, 'r', errors='ignore') as frh:
-            with open(target, 'w') as fwh:
+        with open(source, 'r', encoding='utf-8', errors='ignore') as frh:
+            with open(target, 'w', encoding='utf-8') as fwh:
                 for line in frh.readlines():
                     for key, rules in cfg.items():
                         if line.startswith(key):
@@ -57,7 +64,15 @@ class IFTConverter:
                         fwh.write('\t'.join(chain(*content.values())) + '\n')
 
     def convert_line(self, line, rules):
-        return [line[rule[0]:rule[1]].strip() for rule in rules]
+        return [self.convert_field(line, rule) for rule in rules]
+
+    def convert_field(self, line, rule):
+        field = line[rule[0]:rule[1]].strip()
+        if self.is_number.search(field):
+            field = field.replace(',', '')
+            if field[-1] == '-':
+                field = '-' + field[:-1]
+        return field
 
     def conversion_config(self, filename):
         for key, cfg in self.config.items():
@@ -69,8 +84,8 @@ class IFTConverter:
 
 
 def main():
-    ift_conv = IFTConverter('/home/robert/projekte/python/dbtools/config/IFT/')
-    ift_conv.convert_dir('/home/robert/projekte/python/BMW_Wien/IFT/prod')
+    ift_conv = IFTConverter('E:\\GlobalCube\\Tasks\\Import\\config\\IFT\\')
+    ift_conv.convert_dir('E:\\IFT\\prod')
 
 
 if __name__ == '__main__':

tools/database/ift_import.py

@@ -3,8 +3,8 @@ import subprocess
 
 cfg = {
     'schema': 'temp',
-    'scripts_dir': 'C:/GlobalCube/Tasks/scripts',
-    'query_dir': 'C:/GlobalCube/System/NAVISION/SQL/exec',
+    'scripts_dir': 'E:\\GlobalCube\\Tasks\\scripts',
+    'query_dir': 'E:\\GlobalCube\\System\\NAVISION\\SQL\\exec',
 }
 target_tables = {
     'A': 'IFT_Actuals',
@@ -16,31 +16,30 @@ target_tables = {
 
 
 def task(args):
-    print(args)
-    # return subprocess.Popen(args, stdout=subprocess.DEVNULL).wait()
-    return 0
+    return subprocess.Popen(args, stdout=subprocess.DEVNULL).wait()
 
 
 def ift_import(csv_file, target_table):
-    bulkcopy = str(Path(cfg['scripts_dir']) / 'bulkcopy_import.bat')
+    bulkcopy = str(Path(cfg['scripts_dir']) / 'bcp_import.bat')
     sqlquery = str(Path(cfg['scripts_dir']) / 'sqlquery.bat')
     sqlexec = str(Path(cfg['scripts_dir']) / 'sqlexec.bat')
-    query_file = str(Path(cfg['query_dir']) / (target_table + '.sql'))
+    query_file = target_table + '.sql'
 
     task(f"{sqlquery} \"TRUNCATE TABLE [{cfg['schema']}].[{target_table}]\" ")
     task(f"{bulkcopy} [{cfg['schema']}].[{target_table}] \"{csv_file}\" ")
-    task(f"{sqlexec} {query_file}")
+    task(f"{sqlexec} {query_file} \"{csv_file}\" ")
 
 
 def import_dir(base_dir=None):
     if base_dir is None:
-        base_dir = '/home/robert/projekte/python/BMW_Wien/IFT/prod_csv'
+        base_dir = 'E:\\IFT\\staging'
 
     for csv_file in Path(base_dir).glob('*.csv'):
+        print(csv_file.name)
         if csv_file.name.count('_') == 2:
             _, filetype, _ = csv_file.name.split('_')
             ift_import(str(csv_file), target_tables[filetype])
-            # csv_file.unlink()
+            csv_file.unlink()
 
 
 if __name__ == '__main__':