autoline Kleinigkeiten angepasst, Problem mit Sonderzeichen

import: date-filter2

CARLO.csv

@@ -4,18 +4,18 @@ source;target;filter;query;iterative
 {0}$Archiv_ Filialauftragskopf;Archiv_Filialauftragskopf;;;
 {0}$Archiv_ Filialauftragszeile;Archiv_Filialauftragszeile;;;
 {0}$Archiv_ Werkstattkopf;Archiv_Werkstattkopf;T1.[Buchungsdatum] >= {1};;
-{0}$Archiv_ Werkstattzeile;Archiv_Werkstattzeile;;SELECT T1.* FROM [{0}$Archiv_ Werkstattzeile] T1 INNER JOIN [{0}$Archiv_ Werkstattkopf] T2 ON T1.[Belegart] = T2.[Belegart] AND T1.[Belegnr_] = T2.[Nr_] WHERE T2.[Buchungsdatum] >= {1};
+{0}$Archiv_ Werkstattzeile;Archiv_Werkstattzeile;;SELECT T1.* FROM [{0}$Archiv_ Werkstattzeile] T1 INNER JOIN [{0}$Archiv_ Werkstattkopf] T2 ON T1.[Belegart] = T2.[Belegart] AND T1.[Belegnr_] = T2.[Nr_] WHERE T2.[Buchungsdatum] >= {2};
 {0}$Archivierter Verkaufskopf;Archiv_Verkaufskopf;T1.[Buchungsdatum] >= {1};;
 {0}$Artikel;Artikel;;;timestamp
 {0}$Artikel Buch_-Blattzeile;Artikel_Buch_Blattzeile;;;
 {0}$Calendar_T;Calendar_T;;;
 {0}$Debitor;Debitor;;;
-{0}$Debitorenposten;Debitorenposten;T1.[Buchungsdatum] >= {1};;timestamp
+{0}$Debitorenposten;Debitorenposten;T1.[Buchungsdatum] >= {2} OR T1.[Offen] = '1';;timestamp
 {0}$Debitorgruppe;Debitorgruppe;;;
 {0}$Employee;Employee;;;
 {0}$Employee_T;Employee_T;;;
 {0}$Fahrzeug;Fahrzeug;;;timestamp
-{0}$Fahrzeug Ausstattung;Fahrzeug_Ausstattung;;;timestamp
+{0}$Fahrzeug Ausstattung;Fahrzeug_Ausstattung;T1.[Ausstattungskennzeichen] = '1' OR T1.[Zeilennr_] = '10000';;timestamp
 {0}$Fahrzeug Hauptgruppe;Fahrzeug_Hauptgruppe;;;
 {0}$Fahrzeug Untergruppe;Fahrzeug_Untergruppe;;;
 {0}$Fahrzeugposten;Fahrzeugposten;T1.[Buchungsdatum] >= {1};;
@@ -56,7 +56,7 @@ source;target;filter;query;iterative
 {0}$Produktbuchungsgrp_;Produktbuchungsgrp;;;
 {0}$Produktklasse;Produktklasse;;;
 {0}$Sachkonto;Sachkonto;;;
-{0}$Sachposten;Sachposten;T1.[Buchungsdatum] >= {1};;timestamp
+{0}$Sachposten;Sachposten;T1.[Buchungsdatum] >= {2};;timestamp
 {0}$Task Acquisition Ledger Entry;Task_Acquisition_Ledger_Entry;T1.[Date] >= {1};;
 {0}$Task Type;Task_Type;;;
 {0}$Teilehandel-Statistik Details;Teilehandel_Statistik_Details;;;
@@ -70,9 +70,9 @@ source;target;filter;query;iterative
 {0}$Verkaufsgutschriftszeile;Verkaufsgutschriftszeile;T1.[Buchungsdatum] >= {1};;
 {0}$Verkaufskopf;Verkaufskopf;;;
 {0}$Verkaufsrechnungskopf;Verkaufsrechnungskopf;T1.[Buchungsdatum] >= {1};;timestamp
-{0}$Verkaufsrechnungszeile;Verkaufsrechnungszeile;T1.[Buchungsdatum] >= {1};;timestamp
+{0}$Verkaufsrechnungszeile;Verkaufsrechnungszeile;T1.[Buchungsdatum] >= {2};;timestamp
 {0}$Verkaufszeile;Verkaufszeile;;;
 {0}$Werkstattkopf;Werkstattkopf;;;
-{0}$Werkstattposten;Werkstattposten;T1.[Buchungsdatum] >= {1};;timestamp
+{0}$Werkstattposten;Werkstattposten;T1.[Buchungsdatum] >= {2};;timestamp
 {0}$Werkstattzeile;Werkstattzeile;;;
 {0}$Wertposten;Wertposten;T1.[Buchungsdatum] >= {1};;timestamp

csv_import.py

@@ -75,6 +75,10 @@ def transform_line (line):
         result.append(e)
     return result
 
+def fix_nulls(s):
+    for line in s:
+        yield line.replace('\0', ' ')
+
 def transform_file(source_csv, template):
     global transform
     transform = json.loads(template)
@@ -84,17 +88,15 @@ def transform_file(source_csv, template):
         print(f"Stage-CSV '{stage_csv.name}' ist bereits aktuell.")
         return False
 
-    print(source_csv)
-    print(stage_csv)
+    print(source_csv.name)
     with open(source_csv, "r", encoding="utf8", newline="") as source_file, open(stage_csv, "w", encoding="utf8", newline="") as target_file:
-        csv_read = csv.reader(source_file, delimiter=",")
+        csv_read = csv.reader(fix_nulls(source_file), delimiter=",")
         csv_write = csv.writer(target_file, delimiter="\t")
 
         next(csv_read) # ignore header
         for cols in csv_read:
             csv_write.writerow(transform_line(cols))
 
-
 def csv_tables (csv_dir, target_tables_ci):
     p = re.compile(r"_\d+$")
     result = []
@@ -131,7 +133,7 @@ def batch (csv_dir, action):
     print("set PYTHON=\"C:\\dev\\Python\\Python38-32\"")
 
     for (table, source_csv) in csv_tables(csv_dir, target_tables_ci):
-        print(f"rem =={table}==")
+        print(f"echo =={table}==")
         stage_csv = Path(f"{source_csv.parent}\\stage\\{source_csv.name}")
         try:
             tf_template = transform_template(target_insp, source_csv, table, target_schema)
@@ -141,8 +143,9 @@ def batch (csv_dir, action):
 
             print(f"bcp.exe [{stage_schema}].[{table}] in \"{stage_csv}\" {conn_params(target_dsn)} -c -C 65001 -e \"{stage_csv}.log\" ")
             pkeys = target_insp.get_pk_constraint(table, schema=target_schema)
-            delete_sql = f"DELETE T1 FROM [{target_schema}].[{table}] T1 INNER JOIN [{temp_schema}].[{table}] T2 ON " + " AND ".join([f"T1.[{col}] = T2.[{col}]" for col in pkeys['constrained_columns']])
-            print(f"sqlcmd.exe {conn_params(target_dsn)} -p -Q \"{delete_sql}\" ")
+            if len(pkeys['constrained_columns']) > 0:
+                delete_sql = f"DELETE T1 FROM [{target_schema}].[{table}] T1 INNER JOIN [{temp_schema}].[{table}] T2 ON " + " AND ".join([f"T1.[{col}] = T2.[{col}]" for col in pkeys['constrained_columns']])
+                print(f"sqlcmd.exe {conn_params(target_dsn)} -p -Q \"{delete_sql}\" ")
             insert_sql = f"INSERT INTO [{target_schema}].[{table}] SELECT * FROM [{temp_schema}].[{table}]"
             print(f"sqlcmd.exe {conn_params(target_dsn)} -p -Q \"{insert_sql}\" ")
             print("")

gcstruct.py

@@ -5,7 +5,14 @@ import re
 from bs4 import BeautifulSoup
 
 
-config = ["SKR51", "Herkunft_KST", "Absatzkanal", "Kostentraeger", "Marke", "Standort", "Manuelle_Konten"]
+config = { 
+    'default': ["SKR51", "Herkunft_KST", "Absatzkanal", "Kostentraeger", "Marke", "Standort", "Manuelle_Konten"],
+    'special': {
+        'Planner': ["Kostenstelle", "Konto_1", "Ebene1", "Ebene2"],
+        'Test': ["Ebene1", "Ebene2"]
+    }
+}
+
 columns = ["Konto_Nr", "Konto_Bezeichnung", "Konto_Art", "Kostenstelle", "STK", "Konto_1", "Konto_2", "Konto_3", "Konto_4", "Konto_5"]
 
 def get_tree(node, parents):
@@ -38,19 +45,19 @@ def split_it(text, index):
 
 def get_structure_and_tree ():
     df = pd.read_csv("gcstruct/Kontenrahmen/Kontenrahmen.csv", sep=";", encoding="ansi", decimal=",", converters={i: str for i in range(0, 200)}) #, index_col="Konto_Nr")
-    for i, structure in enumerate(config):
+    for i, structure in enumerate(config['default']):
         first = i*10 + 1
         df[structure] = df['Ebene'+str(first)] + ";" + df['Ebene'+str(first+1)] + ";" + df['Ebene'+str(first+2)] + ";" + df['Ebene'+str(first+3)] + ";" + df['Ebene'+str(first+4)] + ";" + df['Ebene'+str(first+5)] + ";" + df['Ebene'+str(first+6)] + ";" + df['Ebene'+str(first+7)] + ";" + df['Ebene'+str(first+8)] + ";" + df['Ebene'+str(first+9)]
         df['LetzteEbene' + str(i+1) + '_Nr'] = df[structure].apply(lambda x: split_it(x, 0))
         df['LetzteEbene' + str(i+1) + '_Bez'] = df[structure].apply(lambda x: split_it(x, 1))
 
-    df = df[columns + config + ['LetzteEbene' + str(i+1) + '_Nr' for i in range(len(config))] + ['LetzteEbene' + str(i+1) + '_Bez' for i in range(len(config))]]
+    df = df[columns + config['default'] + ['LetzteEbene' + str(i+1) + '_Nr' for i in range(len(config['default']))] + ['LetzteEbene' + str(i+1) + '_Bez' for i in range(len(config['default']))]]
     json_result = { "Kontenrahmen": df.to_dict("records") }
 
     # df2 = pd.read_csv("gcstruct/Strukturen/Kontenrahmen.csv/SKR51.csv", sep=";", encoding="ansi", decimal=",", converters={i: str for i in range(0, 200)})
     # print(df2.head())
 
-    for i, structure in enumerate(config):
+    for i, structure in enumerate(config['default']):
         tree = ET.parse("gcstruct/Xml/" + structure + ".xml")
         json_result[structure] = get_tree(tree.getroot(), [])
         
@@ -60,16 +67,16 @@ def post_structure_and_tree ():
     json_post = json.load(open("gcstruct/SKR51.json", "r"))
 
     # Kontenrahmen.csv
-    ebenen = ["Ebene" + str(i) for i in range(1, len(config)*10+1)]
+    ebenen = ["Ebene" + str(i) for i in range(1, len(config['default'])*10+1)]
     header = ";".join(columns + ebenen)
-    cols = columns + config
+    cols = columns + config['default']
     with open("gcstruct/Kontenrahmen/Kontenrahmen_out.csv", "w", encoding="ansi") as f:
         f.write(header + "\n")
         for row in json_post['Kontenrahmen']:
             f.write(";".join([row[e] for e in cols]) + "\n")
     # print(header)
     # xml und evtl. Struktur.csv
-    for i, structure in enumerate(config):
+    for i, structure in enumerate(config['default']):
         with open("gcstruct/Strukturen/Kontenrahmen.csv/" + structure + "_out.csv", "w", encoding="ansi") as f:
             f.write(";".join(["Ebene" + str(i*10 + j) for j in range(1, 11)]) + "\n")
             rows = structure_from_tree({"id": ";"*9, "children": json_post[structure] })

import.py

@@ -7,11 +7,12 @@ csv_file = "CARLO.csv"
 clients = {'1': "M und S Fahrzeughandel GmbH" }
 client_db = "1"
 date_filter = "'2018-01-01'"
+date_filter2 = "'2019-01-01'"
 source_dsn = { 'user': "sa", 'pass': "Mffu3011#", 'server': "GC-SERVER1\\GLOBALCUBE", 'database': "DE0017" }
 source_schema = "dbo"
 target_dsn = { 'user': "sa", 'pass': "Mffu3011#", 'server': "GC-SERVER1\\GLOBALCUBE", 'database': "CARLO2" }
 target_schema = "import"
-#stage_dir = "\\\\gc-server1\Austausch\\stage"
+# stage_dir = "\\\\gc-server1\Austausch\\stage"
 stage_dir = "C:\\GlobalCube\\System\\CARLO\\Export\\stage"
 
 def db_import (select_query, source_db, current_table, target_db, target_schema):
@@ -63,12 +64,12 @@ for index, current_table in config.iterrows():
             stage_csv = f"{stage_dir}\\{current_table['target']}_{client_db}.csv"
             
             if not pd.isnull(current_table['query']):
-                select_query = current_table['query'].format(prefix, date_filter)
+                select_query = current_table['query'].format(prefix, date_filter, date_filter2)
             else:
                 select_query = f"SELECT T1.* FROM [{source_schema}].[{source_table}] T1 "
 
             if not pd.isnull(current_table['filter']):
-                select_query += " WHERE " + current_table['filter'].format("", date_filter)
+                select_query += " WHERE " + current_table['filter'].format("", date_filter, date_filter2)
 
             source_insp_cols = source_insp.get_columns(source_table)
             source_columns = set([col['name'] for col in source_insp_cols])