NASA: Abgleich, ob Veränderungen vorliegen

config/nasa_config.crypt

@@ -1 +1 @@
-gAAAAABl1PHLWXCLR5w_Bq2yK8IFevNoG70qxmxAxAO2U1ypbuo8qzrNyjbyT5_5_gLlBW1OHHV3m2cULado_qXIB5bjCvm9RNh8MlO2N-lBXp_b8kZlYut6mB71qnc_swttGaQJJqsdCLQUHBMzq0024o3iISR1sUG6dILGlhMiDu6H2xhrN1Licn1QXRtWRjBnhgazant4Jpt1uEl-nXRpLnKzV8tDyoaMibcWNsvaECWWy3XB7KKtbBhdcaXVcxBKevuoCA1DV4ydzsdfTt609sLr4sYmU1B5UvNWNgtCeZyE75qQejlv1D3Jm8lJ9fpQm5tg9XGn5R5Y6ic_-mWAiYkBTU9UuTFiryJ_KkQjTNQ3ml5jg0wNB-mwV2smPLFsZU5R3frbwo8sfNipC5BB9sijwQG1VeyjOQBKM04EJSO6S0CAHFlOYKDS8JDw6BNvam9PAhnekd1NUePRAh_n83AHS76ZpoRSDMcITJqfx9dJtES6DtIaXD5MGQ62bDEYYLfJFmygysfZHzPHhPKPW2BY4tTGhayyMC0Hl98zmkQJJ_5I5IloPDk7RkN1tPkKPF4fSC8m
+gAAAAABnVrXRCuKVsIsPGDL2D2Q3GZ6A9O67nFqQJNKEfnjFxIctoPqaQtCwRxR6Yl1KWYKJxXXKuanAtjjoS6gPmBRtaUKyKhWlCf6SCtC9i4nLpLrP01dBfJVDESO4I9Er_oUxXEapT4XIWOtRrfp2xVrNr8KjlVFHcHRT8URk2gBMcnrYJzAuRwwuQDYnMsvKKENdhM_0uvM_fYxnsNYNcMR5CgPD1raYp3r3hP91A5DjC9MEH3s0U9TtRqqyDqnf40uF9i6MD0Z1OZAEZZbdC0-pceIRZklr78aPHhuNom0wusZGxqcP23zZ8Po9sDCb3wScF7anQQ3uF6MTfCGs-D2qyKeD2qML-Bo2krJjoSm9Jyy9MWRw8MjUkLZ-dPF9-ZxhglOICIbdvpSqL0xC-Wnq47VOnExVdZ995lV9_MPWSiheGRlIgieVDXq55Ck8-UJT1dKTwHIZDtO0Dnuda0n-zDhNuscCGdk5BomzX6H-Izi0FGXtUdAoUvB-zSClEvpNH6RM6owoALhB_uDMCseuFj2rqQAOvf5f1X0nTxy1snfCjjq5rs0OXHRB0WlkjdfKugJC

config/nasa_config.json

@@ -9,10 +9,10 @@
     "selected_year": "2020",
     "selected_month": "05",
     "source_dsn": { 
-        "server": "GC-SERVER1\\GLOBALCUBE", 
+        "server": "localhost\\GLOBALCUBE", 
         "user": "sa", 
         "pass": "Mffu3011#", 
-        "database": "GAPS", 
+        "database": "Mazda", 
         "schema": "dbo" 
     }
 }

mazda_webservice.py

@@ -102,7 +102,7 @@ def profile():
         return redirect("/")
 
     data = mazda_upload.convert_csv(
-        base_dir + "data/Workshop_Order_Report.csv", base_dir + "temp/mazda_export.json", 2024, 1
+        base_dir + "data/Workshop_Order_Report.csv", base_dir + "temp/mazda_export.json", 2023, 12
     )
     try:
         mazda_upload.upload(oauth, data)

nasa_upload.py

@@ -1,9 +1,12 @@
+import hashlib
 import logging
+import os
+import re
+import shutil
 import pandas as pd
 import json
 from pathlib import Path
-from datetime import date
-from dateutil.relativedelta import relativedelta
+from datetime import datetime, timedelta
 from sqlalchemy import create_engine
 from suds.client import Client
 from cryptography.fernet import Fernet
@@ -28,17 +31,13 @@ def get_config():
     return config
 
 
-def conn_string(dsn):
+def conn_string(dsn: dict[str, str]):
     return f"mssql+pyodbc://{dsn['user']}:{dsn['pass']}@{dsn['server']}/{dsn['database']}?driver=SQL+Server+Native+Client+11.0"
 
 
-def load_data(config, source, year=None, month=None):
-    if year is None:
-        curr_date = date.today() - relativedelta(months=+1)
-        year = curr_date.strftime("%Y")
-        month = curr_date.strftime("%m")
-    period = f"{year}-{month}"
-    period2 = int(f"{year}{month}")
+def load_data(config: dict[str, str], source: str, period: str):
+    year = period[:4]
+    month = period[4:6]
 
     select_befehl_auftraege = f"SELECT * FROM [Auftraege_NASA_gruppiert] WHERE Periode = '{period}'"
     select_befehl_mitarbeiter = f"SELECT * FROM [Mitarbeiter_NASA] WHERE Periode = '{period}'"
@@ -61,12 +60,16 @@ def load_data(config, source, year=None, month=None):
     if source == "database":
         source_db = create_engine(conn_string(config["source_dsn"]))
         df = pd.read_sql(select_befehl_auftraege, con=source_db)
+        rename_from = ["AuftragsArt", "AuftragsTyp"]
+        rename_to = ["AuftragsArtId_Name", "AuftragsArt"]
+        df = df.rename(columns=dict(zip(rename_from, rename_to)))
+
     else:
         df = pd.read_csv(source_auftraege, sep=";", encoding="latin-1", decimal=",")
-        df = df[df["Periode"] == period2]
+        df = df[df["Periode"] == period]
 
-    # auftragsart = ["Extern", "Garantie", "Intern", "Theke"]
-    # auftragstyp = ["Inspektion", "Karosseriearbeit", "Lackierung", "Verschleißteile", "Sonstiges"]
+    # AuftragsArt = ["Inspektion", "Karosseriearbeit", "Lackierung", "Verschleißteile", "Sonstiges"]
+    # AuftragsArtId = {"1": "Extern", "2": "Garantie", "3": "Intern", "4": "Theke"]
 
     columns = [
         "AuftragsArt",
@@ -81,14 +84,14 @@ def load_data(config, source, year=None, month=None):
     df = df[columns]
 
     df.to_csv(
-        f"export/{period}_auftraege.csv",
+        f"{config['export_dir']}/csv/{period}_auftraege.csv",
         sep=";",
         encoding="latin-1",
         decimal=",",
         index=False,
     )
 
-    payload["WerkstattDurchlaeufe"] = df["AnzahlAuftraege"].sum()
+    payload["WerkstattDurchlaeufe"] = int(df["AnzahlAuftraege"].sum())
     payload["AfterSalesPositionen"] = df.to_dict("records")
 
     # Mitarbeiter gesamt und produktiv
@@ -98,7 +101,7 @@ def load_data(config, source, year=None, month=None):
         df = pd.read_csv(source_mitarbeiter, sep=";", encoding="latin-1", decimal=",")
 
     df.to_csv(
-        f"export/{period}_mitarbeiter.csv",
+        f"{config['export_dir']}/csv/{period}_mitarbeiter.csv",
         sep=";",
         encoding="latin-1",
         decimal=",",
@@ -106,11 +109,11 @@ def load_data(config, source, year=None, month=None):
     )
 
     payload["AnzahlMitarbeiter"] = df.shape[0]
-    payload["AnzahlProduktiv"] = df["Prod"].sum()
+    payload["AnzahlProduktiv"] = int(df["produktiv"].sum())
     return payload
 
 
-def submit_data(config, payload):
+def submit_data(config: dict[str, str], payload):
     client = Client(
         url=config["service_url"],
         username=config["credentials"]["username"],
@@ -124,7 +127,7 @@ def submit_data(config, payload):
     return -1
 
 
-def print_result(period, result, len_pos):
+def print_result(period: str, result: str, len_pos: int):
     print("Periode: " + period)
     if len_pos == result:
         print(f"Erfolgreich {result} Datensätze übertragen")
@@ -137,20 +140,85 @@ def print_result(period, result, len_pos):
         print(f"{len_pos - result} von {len_pos} Datensätzen nicht verarbeitet!")
 
 
-def workflow(config, year, month):
-    period = f"{year}-{month}"
-    payload = load_data(config, "csv", year, month)
+def workflow(config: dict[str, str], year, month):
+    period = f"{year}{month}"
+    payload = load_data(config, "csv", period)
     result = submit_data(config, payload)
     len_pos = len(payload["AfterSalesPositionen"])
     print_result(period, result, len_pos)
 
 
+def export_all_periods(config) -> None:
+    dt = datetime.now()
+    prev = str(dt.year - 1)
+    periods = [f"{prev}{x:02}" for x in range(1, 13)] + [f"{dt.year}{x:02}" for x in range(1, dt.month)]
+
+    for period in periods:
+        payload = load_data(config, "database", period)
+        json.dump(
+            payload,
+            open(f"export/NASA/temp/NASA_{config['client_id']}_{period}_{config['timestamp']}.json", "w"),
+            indent=2,
+        )
+
+
+def file_get_hash(filename: str) -> str:
+    with open(filename, "r") as frh:
+        data = frh.read()
+        return calculate_sha256(data)
+
+
+def calculate_sha256(data: str) -> str:
+    return hashlib.sha256(data.encode()).hexdigest()
+
+
+def archive_files(export_dir: str):
+    last_week = (datetime.now() - timedelta(days=6)).timestamp()
+    for file in Path(export_dir).glob("*.json"):
+        if file.stat().st_ctime < last_week:
+            file.unlink()
+
+    archive_path = Path(export_dir + "/Archiv")
+    for file in Path(export_dir + "/temp").glob("*.json"):
+        p = re.search(r"NASA_\d{5}_(20\d{4})_", file.name)
+        if not p:
+            continue
+        period = p[1]
+        year = period[:4]
+        dest_folder = archive_path / year / period
+        os.makedirs(dest_folder, exist_ok=True)
+        file_hash = file_get_hash(file)
+
+        if has_identical_file(dest_folder, file_hash):
+            file.unlink()
+            continue
+        shutil.copy(file, archive_path.parent / file.name)
+        file.rename(dest_folder / file.name)
+
+
+def has_identical_file(target: Path, file_hash: str) -> bool:
+    for archived_file in Path(target).glob("*.json"):
+        if file_get_hash(archived_file) == file_hash:
+            return True
+    return False
+
+
+def submit_changes(config):
+    for file in Path(config["export_dir"] + "/temp").glob("NASA_*.json"):
+        payload = json.load(file.open("r"))
+        period = payload["Jahr"] + payload["Monat"]
+        len_pos = len(payload["AfterSalesPositionen"])
+        result = submit_data(config, payload)
+        print_result(period, result, len_pos)
+
+
 def main():
     config = get_config()
-    year = "2023"
-    for p in range(1, 13):
-        month = f"{p:02}"
-        workflow(config, year, month)
+    config["timestamp"] = datetime.now().strftime("%Y%m%d_%H%M%S")
+    config["export_dir"] = str(Path(".").resolve() / "export" / "NASA")
+    export_all_periods(config)
+    archive_files(config["export_dir"])
+    submit_changes(config)
 
 
 if __name__ == "__main__":

queries/Auftraege_NASA_gruppiert.sql

@@ -0,0 +1,115 @@
+WITH
+    [Auftraege]
+        AS (SELECT [T1].[INVOICE_DATE]
+                 , [T1].[ORDER_NUMBER]
+                 , [T1].[STATUS]
+                 , [T1].[CUSTOMER_GROUP]
+                 , [T1].[DEPARTMENT]
+                 , [T2].[ORDER_LINETYPE]
+                 , [T2].[LINES_NET_VALUE]
+                 , [T2].[REDUCTION_AMOUNT]
+                 , [T2].[REPAIR_GROUP]
+                 , [T2].[INV_TIME]
+                 , [T2].[PROD_CODE]
+            FROM [dbo].[ORDER_HEADER] [T1]
+                     LEFT OUTER JOIN [dbo].[ORDER_LINE] [T2] ON [T1].[ORDER_NUMBER] = [T2].[ORDER_NUMBER]
+            WHERE [T1].[STATUS] IN ('34', '35', '36', '37', '39', '47', '49', '50', '51')
+              AND [T1].[INVOICE_DATE] >= convert(datetime, '2023-01-01 00:00:00.000')
+              AND left([T1].[DEPARTMENT], 2) IN ('03'))
+  , [Auftraege_Auftragsart]
+        AS (SELECT [INVOICE_DATE]
+                 , [ORDER_NUMBER]
+                 , [STATUS]
+                 , [CUSTOMER_GROUP]
+                 , [DEPARTMENT]
+                 , [ORDER_LINETYPE]
+                 , [LINES_NET_VALUE]
+                 , [REDUCTION_AMOUNT]
+                 , rtrim([REPAIR_GROUP]) AS [REPAIR_GROUP]
+                 , [INV_TIME]
+                 , [PROD_CODE]
+                 , year([INVOICE_DATE]) * 100 + month([INVOICE_DATE]) AS [Periode]
+                 , substring([DEPARTMENT], 4, 1) AS [Kostenstelle]
+                 , CASE
+                       WHEN ([STATUS] BETWEEN '30' AND '39') THEN 'Theke'
+                       WHEN (([LINES_NET_VALUE] = .00) AND ([REDUCTION_AMOUNT] <> .00)) THEN 'intern'
+                       WHEN ([CUSTOMER_GROUP] LIKE '6%') THEN 'Garantie'
+                       WHEN ([CUSTOMER_GROUP] LIKE '9%') THEN 'intern'
+                       WHEN (left([CUSTOMER_GROUP], 1) BETWEEN 'A' AND 'Z') THEN 'intern'
+                       WHEN ([CUSTOMER_GROUP] IN ('00')) THEN 'intern'
+                       ELSE 'extern' END AS [AuftragsArt]
+            FROM [Auftraege] [T1])
+  , [Auftraege_Auftragstyp]
+        AS (SELECT *
+                 , CASE
+                       WHEN ([Kostenstelle] IN ('3') OR [AuftragsArt] = 'Theke') THEN 'Sonstiges'
+                       WHEN ([REPAIR_GROUP] NOT IN ('2', '3', '4')) THEN 'Inspektion'
+                       WHEN ([REPAIR_GROUP] IN ('2')) THEN 'Karosseriearbeit'
+                       WHEN ([REPAIR_GROUP] IN ('4')) THEN 'Lackierung'
+                       ELSE 'Sonstiges' END AS [AuftragsTyp]
+            FROM [Auftraege_Auftragsart])
+  , [Auftraege_Linetype]
+        AS (SELECT [Periode]
+                 , [AuftragsArt]
+                 , CASE
+                       WHEN ([AuftragsArt] IN ('extern')) THEN '1'
+                       WHEN ([AuftragsArt] IN ('Garantie')) THEN '2'
+                       WHEN ([AuftragsArt] IN ('intern')) THEN '3'
+                       WHEN ([AuftragsArt] IN ('Theke')) THEN '4'
+            END AS [AuftragsartId]
+                 , [AuftragsTyp]
+                 , CASE
+                       WHEN ([AuftragsTyp] IN ('Inspektion')) THEN '1'
+                       WHEN ([AuftragsTyp] IN ('Karosseriearbeit')) THEN '2'
+                       WHEN ([AuftragsTyp] IN ('Lackierung')) THEN '3'
+                       WHEN ([AuftragsTyp] IN (N'Verschleißteile')) THEN '4'
+                       ELSE '5' END AS [AuftragsTypId]
+                 , [ORDER_NUMBER]
+                 , iif([ORDER_LINETYPE] = '1', [LINES_NET_VALUE], 0.00) AS [Teile]
+                 , iif([ORDER_LINETYPE] = '4', iif([LINES_NET_VALUE] <> 0.00, [LINES_NET_VALUE], [REDUCTION_AMOUNT]), 0.00) AS [Lohn]
+                 , iif([ORDER_LINETYPE] = '3', [LINES_NET_VALUE], 0.00) AS [Sonstige]
+                 , iif([ORDER_LINETYPE] NOT IN ('1', '3', '4'), [LINES_NET_VALUE], 0.00) AS [Rest]
+            FROM [Auftraege_Auftragstyp])
+  , [Auftraege_gruppiert]
+        AS (SELECT [Periode]
+                 , [AuftragsArt]
+                 , [AuftragsartId]
+                 , [AuftragsTyp]
+                 , [AuftragsTypId]
+                 , [ORDER_NUMBER]
+                 , sum([Teile]) AS [TeileUmsatz]
+                 , sum([Lohn]) AS [LohnUmsatz]
+                 , sum([Sonstige]) AS [SonstigeUmsatz]
+                 , sum([Teile] + [Lohn] + [Sonstige]) AS [GesamtUmsatz]
+                 , sum([Rest]) AS [RestUmsatz]
+                 , 1 AS [AnzahlAuftraege]
+            FROM [Auftraege_Linetype]
+            GROUP BY [Periode]
+                   , [AuftragsArt]
+                   , [AuftragsartId]
+                   , [AuftragsTyp]
+                   , [AuftragsTypId]
+                   , [ORDER_NUMBER])
+  , [Auftraege_kumuliert]
+        AS (SELECT [Periode]
+                 , [AuftragsArt]
+                 , [AuftragsartId]
+                 , [AuftragsTyp]
+                 , [AuftragsTypId]
+                 , sum([TeileUmsatz]) AS [TeileUmsatz]
+                 , sum([LohnUmsatz]) AS [LohnUmsatz]
+                 , sum([SonstigeUmsatz]) AS [SonstigeUmsatz]
+                 , sum([GesamtUmsatz]) AS [GesamtUmsatz]
+                 , sum([RestUmsatz]) AS [RestUmsatz]
+                 , sum([AnzahlAuftraege]) AS [AnzahlAuftraege]
+            FROM [Auftraege_gruppiert]
+            GROUP BY [Periode]
+                   , [AuftragsArt]
+                   , [AuftragsartId]
+                   , [AuftragsTyp]
+                   , [AuftragsTypId])
+
+SELECT *
+--INTO [Auftraege_NASA_gruppiert]
+FROM [Auftraege_kumuliert]
+ORDER BY [Periode], [AuftragsArtId], [AuftragsTypId]

queries/Mitarbeiter_NASA.sql

@@ -0,0 +1,51 @@
+WITH
+    "Jahre"
+        AS (SELECT DISTINCT ([T1].[number] + 2000) AS "Jahr"
+            FROM [master].[dbo].[spt_values] [T1]
+            WHERE ([T1].[number] + 2000) BETWEEN '2020' AND year(getdate())
+              AND [T1].[type] = 'P')
+  , "Monate"
+        AS (SELECT DISTINCT [T1].[number] AS "Monat"
+            FROM [master].[dbo].[spt_values] [T1]
+            WHERE ([T1].[number] BETWEEN 1 AND 12)
+              AND [T1].[type] = 'P')
+  , "Periode"
+        AS (SELECT [Jahr] * 100 + [Monat] AS [Periode]
+                 , datefromparts([Jahr], [Monat], 1) AS [Datum]
+            FROM [Jahre]
+                     CROSS JOIN [Monate])
+  , "Punch_min_max"
+        AS (SELECT [PROFILE_CODE]
+                 , min([START_DATE_TIME]) AS [START_DATE]
+                 , max([END_DATE_TIME]) AS [END_DATE]
+            FROM [dbo].[PUNCH]
+			WHERE left([DONE_FOR_DEPARTMENT], 2) IN ('03')
+            GROUP BY [PROFILE_CODE])
+  , "Punch_produktiv"
+        AS (SELECT DISTINCT [PROFILE_CODE]
+                          , year([START_DATE_TIME]) * 100 + month([START_DATE_TIME]) AS [Periode]
+                          , 1 AS [produktiv]
+            FROM [dbo].[PUNCH]
+            WHERE [ACTIVITY_CODE] = '1060')
+  , "Mitarbeiter_aktiv"
+        AS (SELECT [T3].[PERSON_ID]
+                 , [T3].[JOB_START_DATE]
+                 , [T3].[JOB_END_DATE]
+                 , [T3].[PROFESSION_GROUP_ID]
+                 , [T3].[WORK_LEADER_GROUP_ID]
+                 , [T3].[EMPLOYEE_GROUP]
+                 , [T2].*
+            FROM [dbo].[PROFILE] [T1]
+                     INNER JOIN [dbo].[EMPLOYEE] [T3] ON [T1].[PERSON_ID] = [T3].[PERSON_ID]
+                     LEFT JOIN [Punch_min_max] [T2] ON [T1].[PROFILE_CODE] = [T2].[PROFILE_CODE])
+  , "Mitarbeiter_Periode"
+        AS (SELECT *
+            FROM [Mitarbeiter_aktiv] [T1]
+                     CROSS JOIN [Periode] [T2]
+            WHERE [T2].[Datum] BETWEEN [T1].[START_DATE] AND [T1].[END_DATE])
+
+SELECT [T1].*
+     , isnull([T2].[produktiv], 0) AS [produktiv]
+--INTO Mitarbeiter_NASA
+FROM "Mitarbeiter_Periode" [T1]
+         LEFT JOIN "Punch_produktiv" [T2] ON [T1].[PROFILE_CODE] = [T2].[PROFILE_CODE] AND [T1].[Periode] = [T2].[Periode]

queries/test.sql

@@ -296,12 +296,12 @@ ELSE T1."Verk_ an Deb_-Nr_"
 END like ' % INT % ' or CASE WHEN T1."Rech_ an Deb_-Nr_" is null THEN T2."Rech_ an Deb_-Nr_"
                         ELSE T1."Rech_ an Deb_-Nr_"
 END like ' % INT % ' THEN ' Intern '
-WHEN T3."Belegtyp" = 2 AND(T3."Herkunftsnr_" NOT like ' % INT % ' orCASE WHEN T1."Verk_ an Deb_-Nr_" is null THEN T2."Verk_ an Deb_-Nr_"
+WHEN T3."Belegtyp" = 2 AND(T3."Herkunftsnr_" NOT like ' % INT % ' or CASE WHEN T1."Verk_ an Deb_-Nr_" is null THEN T2."Verk_ an Deb_-Nr_"
                         ELSE T1."Verk_ an Deb_-Nr_"
 END NOT like ' % INT % ' or CASE WHEN T1."Rech_ an Deb_-Nr_" is null THEN T2."Rech_ an Deb_-Nr_"
                                                     ELSE T1."Rech_ an Deb_-Nr_"
             END NOT like ' % INT % ') THEN ' Extern '
-WHEN T3."Belegtyp" = 3 AND(T3."Herkunftsnr_" NOT like ' % INT % ' orCASE WHEN T1."Verk_ an Deb_-Nr_" is null THEN T2."Verk_ an Deb_-Nr_"
+WHEN T3."Belegtyp" = 3 AND(T3."Herkunftsnr_" NOT like ' % INT % ' or CASE WHEN T1."Verk_ an Deb_-Nr_" is null THEN T2."Verk_ an Deb_-Nr_"
                         ELSE T1."Verk_ an Deb_-Nr_"
 END NOT like ' % INT % ' or CASE WHEN T1."Rech_ an Deb_-Nr_" is null THEN T2."Rech_ an Deb_-Nr_"
                                                     ELSE T1."Rech_ an Deb_-Nr_"