Anpassungen für Produktivsystem

mazda_archive.py

@@ -0,0 +1,62 @@
+import hashlib
+import logging
+import os
+import re
+import shutil
+from datetime import datetime, timedelta
+from pathlib import Path
+
+logger = logging.getLogger("mazda")
+
+
+def file_get_hash(filename: str) -> str:
+    with open(filename, "r") as frh:
+        data = frh.read()
+        return calculate_sha256(data)
+
+
+def calculate_sha256(data: str) -> str:
+    return hashlib.sha256(data.encode()).hexdigest()
+
+
+def archive_files(export_dir: str):
+    last_week = (datetime.now() - timedelta(days=6)).timestamp()
+    for file in Path(export_dir).glob("*.json"):
+        if file.stat().st_ctime < last_week:
+            file.unlink()
+
+    archive_path = Path(export_dir + "/Archiv")
+    for file in Path(export_dir + "/temp").glob("*.json"):
+        p = re.search(r"order-report_(\d*)_(20\d{4})_", file.name)
+        if not p:
+            continue
+        order_no = p[1]
+        period = p[2]
+        year = period[:4]
+        dest_folder = archive_path / year / period
+        os.makedirs(dest_folder, exist_ok=True)
+        file_hash = file_get_hash(file)
+
+        if has_identical_file(dest_folder, file_hash, order_no):
+            file.unlink()
+            continue
+        shutil.copy(file, archive_path.parent / file.name)
+        file.rename(dest_folder / file.name)
+
+
+def has_identical_file(target: Path, file_hash: str, order_no: str) -> bool:
+    for archived_file in Path(target).glob("*.json"):
+        if order_no not in archived_file.name:
+            continue
+        if file_get_hash(archived_file) == file_hash:
+            return True
+    return False
+
+
+def main():
+    base_dir = "C:/projekte/mazda/"
+    archive_files(base_dir + "export/Mazda")
+
+
+if __name__ == "__main__":
+    main()

mazda_export.py

@@ -0,0 +1,191 @@
+import json
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+from dateutil.relativedelta import relativedelta
+
+base_dir = "C:/projekte/mazda/"
+
+
+def date_format(d: datetime):
+    if d == 0:
+        return ""  # '0000-00-00T00:00:00.000Z'
+    date_str = d.isoformat(sep="T")
+    if len(date_str) == 19:
+        return date_str + ".000Z"
+    return date_str[:-3] + "Z"
+
+
+def convert_csv(import_csv, export_json, year, month):
+    date_min = datetime(year, month, 5, 0, 0, 0)
+    date_max = datetime(year, month, 5, 0, 0, 0) + relativedelta(days=5)
+
+    date_cols = ["invoiceDate", "orderDate", "orderCompletionDate", "vehicleIntakeDate", "nextMotDueDate"]
+    df = pd.read_csv(import_csv, encoding="latin-1", decimal=",", sep=";", parse_dates=date_cols)
+    df = df.fillna(0)  # [(df["invoiceDate"] >= date_min) & (df["invoiceDate"] <= date_max)]
+    df = df.sort_values(by=["invoiceDate", "invoiceNumber", "orderNumber", "lineNumber"])
+    df["vin"] = np.where(df["vin"] == 0, "0" * 17, df["vin"])
+    # print(df[['currency','documentType','invoiceCategory','invoiceDate','invoiceNumber']].drop_duplicates().info())
+    invoices_filter = ["currency", "documentType", "invoiceCategory", "invoiceDate", "invoiceNumber"]
+    invoices = df[invoices_filter].drop_duplicates().to_dict("records")
+    invoice_items_filter = [
+        "invoiceNumber",
+        "orderLineNumber",
+        "orderNumber",
+        "amount",
+        "discount",
+        "portion",
+        "unitPrice",
+    ]
+    invoice_items = df[invoice_items_filter].groupby("invoiceNumber")
+
+    for invoice in invoices:
+        invoice["invoiceDate"] = date_format(invoice["invoiceDate"])
+        items = invoice_items.get_group(invoice["invoiceNumber"])
+        items.pop("invoiceNumber")
+        invoice["invoiceItems"] = items.to_dict("records")
+
+    orders = (
+        df[["orderNumber", "orderDate", "orderCompletionDate", "vehicleIntakeDate"]]
+        .drop_duplicates()
+        .to_dict("records")
+    )
+    orders_vehicle_filter = ["orderNumber", "licensePlate", "nextMotDueDate", "odometer", "odometerUnit", "vin"]
+    orders_vehicle = df[orders_vehicle_filter].drop_duplicates().groupby("orderNumber")
+    orders_items = (
+        df[
+            [
+                "orderNumber",
+                "lineNumber",
+                "orderItemType",
+                "category",
+                "descriptionOperation",
+                "hours",
+                "operationCode",
+                "standardHours",
+                "descriptionOther",
+                "type",
+                "descriptionPart",
+                "isDamageCausal",
+                "manufacturer",
+                "partNumber",
+                "quantity",
+                "serialNumber",
+                "unit",
+                "company",
+                "descriptionPurchase",
+                "invoiceCode",
+                "invoiceDate",
+                "invoiceNumber",
+            ]
+        ]
+        .drop_duplicates()
+        .groupby("orderNumber")
+    )
+
+    for order in orders:
+        order["vehicle"] = orders_vehicle.get_group(order["orderNumber"]).to_dict("records")[0]
+        order["vehicle"]["nextMotDueDate"] = date_format(order["vehicle"]["nextMotDueDate"])
+        # odo_str = str(order["vehicle"]["odometer"])
+        order["vehicle"]["odometer"] = int(order["vehicle"]["odometer"])
+
+        order["orderDate"] = date_format(order["orderDate"])
+        order["orderCompletionDate"] = date_format(order["orderCompletionDate"])
+        order["vehicleIntakeDate"] = date_format(order["vehicleIntakeDate"])
+
+        items = orders_items.get_group(order["orderNumber"]).to_dict("records")
+        order["items"] = []
+        for item in items:
+            if item["orderItemType"] == "operation":
+                order["items"].append(
+                    {
+                        "lineNumber": item["lineNumber"],
+                        "operation": {
+                            "category": item["category"],
+                            "description": item["descriptionOperation"],
+                            "hours": item["hours"],
+                            "operationCode": item["operationCode"],
+                            "standardHours": item["standardHours"],
+                        },
+                    }
+                )
+            elif item["orderItemType"] == "part":
+                order["items"].append(
+                    {
+                        "lineNumber": item["lineNumber"],
+                        "part": {
+                            "description": item["descriptionPart"],
+                            "isDamageCausal": item["isDamageCausal"],
+                            "manufacturer": item["manufacturer"],
+                            "partNumber": item["partNumber"],
+                            "quantity": item["quantity"],
+                            "serialNumber": str(item["serialNumber"]),
+                            "unit": "pcs",
+                        },
+                    }
+                )
+            elif item["orderItemType"] == "other":
+                order["items"].append(
+                    {
+                        "lineNumber": item["lineNumber"],
+                        "other": {"description": item["descriptionOther"], "type": item["type"]},
+                    }
+                )
+            else:
+                order["items"].append(
+                    {
+                        "lineNumber": item["lineNumber"],
+                        "purchaseInvoice": {
+                            "company": item["company"],
+                            "description": item["descriptionPurchase"],
+                            "invoiceCode": item["invoiceCode"],
+                            "invoiceDate": date_format(item["invoiceDate"]),
+                            "invoiceNumber": item["invoiceNumber"],
+                        },
+                    }
+                )
+
+    res = {
+        "creationDate": date_format(datetime.now()),
+        "invoices": invoices,
+        "orders": orders,
+        "timeRangeBegin": date_format(date_min),
+        "timeRangeEnd": date_format(date_max),
+    }
+
+    # json.dump(res, open(export_json, "w"), indent=2)
+    return res
+
+
+def export_orders(data, export_dir):
+    dt = datetime.now()
+    timestamp = dt.strftime("%Y%m%d_%H%M%S")
+
+    invoices = data["invoices"]
+    orders = data["orders"]
+
+    data["invoices"] = []
+    data["orders"] = []
+
+    for order in orders:
+        order_date = order["orderDate"]
+        data["creationDate"] = order_date
+        data["timeRangeBegin"] = order_date
+        data["timeRangeEnd"] = order_date
+        period = order_date[:4] + order_date[5:7]
+        data["orders"] = [order]
+        order_no = order["orderNumber"]
+        data["invoices"] = [i for i in invoices if i["invoiceItems"][0]["orderNumber"] == order_no]
+        json.dump(data, open(f"{export_dir}/order-report_{order_no}_{period}_{timestamp}.json", "w"), indent=2)
+
+
+def main():
+    data = convert_csv(
+        base_dir + "data/Workshop_Order_Report.csv", base_dir + "export/Mazda/temp/mazda_export.json", 2024, 1
+    )
+    export_orders(data, base_dir + "export/Mazda/temp")
+
+
+if __name__ == "__main__":
+    main()

mazda_upload.py

@@ -1,12 +1,11 @@
 import json
 from dataclasses import dataclass
-from datetime import datetime
+from pathlib import Path
 
-import numpy as np
-import pandas as pd
-from dateutil.relativedelta import relativedelta
 from requests_oauthlib import OAuth2Session
 
+from mazda_export import convert_csv
+
 
 @dataclass
 class MazdaConfig:
@@ -22,7 +21,20 @@ class MazdaConfig:
     dealer_number: str
 
 
-cfg = MazdaConfig(
+cfg_prod = MazdaConfig(
+    domain="https://mapps.mazdaeur.com",
+    webservice="/dogma-restapi-dms/api",
+    module="/vehicles/workshop/order-report",
+    auth_url="/oauth/authorize",
+    token_url="/oauth/token",
+    client_id="2F6FACF0-F216-FB26-868A-BAAF82890E62",
+    client_secret="8scMw?34ecQT)M=7!C=LQ",
+    username="mmd15972.globalcube2",
+    password="fBYxJ!nfj)4?E3N",
+    dealer_number="15972/MMD",
+)
+
+cfg_debug = MazdaConfig(
     domain="https://mappsacc.mazdaeur.com",
     webservice="/dogma-restapi-dms/api",
     module="/vehicles/workshop/order-report",
@@ -35,161 +47,12 @@ cfg = MazdaConfig(
     dealer_number="15972/MMD",
 )
 
+
 redirect_uri = "https://localhost:8085/"
 base_dir = "C:/projekte/mazda/"
 
 
-def date_format(d: datetime):
-    if d == 0:
-        return ""  # '0000-00-00T00:00:00.000Z'
-    date_str = d.isoformat(sep="T")
-    if len(date_str) == 19:
-        return date_str + ".000Z"
-    return date_str[:-3] + "Z"
-
-
-def convert_csv(import_csv, export_json, year, month):
-    date_min = datetime(year, month, 5, 0, 0, 0)
-    date_max = datetime(year, month, 5, 0, 0, 0) + relativedelta(days=5)
-
-    date_cols = ["invoiceDate", "orderDate", "orderCompletionDate", "vehicleIntakeDate", "nextMotDueDate"]
-    df = pd.read_csv(import_csv, encoding="latin-1", decimal=",", sep=";", parse_dates=date_cols)
-    df = df.fillna(0)[(df["invoiceDate"] >= date_min) & (df["invoiceDate"] <= date_max)]
-    df = df.sort_values(by=["invoiceDate", "invoiceNumber", "orderNumber", "lineNumber"])
-    df["vin"] = np.where(df["vin"] == 0, "0" * 17, df["vin"])
-    # print(df[['currency','documentType','invoiceCategory','invoiceDate','invoiceNumber']].drop_duplicates().info())
-    invoices_filter = ["currency", "documentType", "invoiceCategory", "invoiceDate", "invoiceNumber"]
-    invoices = df[invoices_filter].drop_duplicates().to_dict("records")
-    invoice_items_filter = [
-        "invoiceNumber",
-        "orderLineNumber",
-        "orderNumber",
-        "amount",
-        "discount",
-        "portion",
-        "unitPrice",
-    ]
-    invoice_items = df[invoice_items_filter].groupby("invoiceNumber")
-
-    for invoice in invoices:
-        invoice["invoiceDate"] = date_format(invoice["invoiceDate"])
-        items = invoice_items.get_group(invoice["invoiceNumber"])
-        items.pop("invoiceNumber")
-        invoice["invoiceItems"] = items.to_dict("records")
-
-    orders = (
-        df[["orderNumber", "orderDate", "orderCompletionDate", "vehicleIntakeDate"]]
-        .drop_duplicates()
-        .to_dict("records")
-    )
-    orders_vehicle_filter = ["orderNumber", "licensePlate", "nextMotDueDate", "odometer", "odometerUnit", "vin"]
-    orders_vehicle = df[orders_vehicle_filter].drop_duplicates().groupby("orderNumber")
-    orders_items = (
-        df[
-            [
-                "orderNumber",
-                "lineNumber",
-                "orderItemType",
-                "category",
-                "descriptionOperation",
-                "hours",
-                "operationCode",
-                "standardHours",
-                "descriptionOther",
-                "type",
-                "descriptionPart",
-                "isDamageCausal",
-                "manufacturer",
-                "partNumber",
-                "quantity",
-                "serialNumber",
-                "unit",
-                "company",
-                "descriptionPurchase",
-                "invoiceCode",
-                "invoiceDate",
-                "invoiceNumber",
-            ]
-        ]
-        .drop_duplicates()
-        .groupby("orderNumber")
-    )
-
-    for order in orders:
-        order["vehicle"] = orders_vehicle.get_group(order["orderNumber"]).to_dict("records")[0]
-        order["vehicle"]["nextMotDueDate"] = date_format(order["vehicle"]["nextMotDueDate"])
-        # odo_str = str(order["vehicle"]["odometer"])
-        order["vehicle"]["odometer"] = int(order["vehicle"]["odometer"])
-
-        order["orderDate"] = date_format(order["orderDate"])
-        order["orderCompletionDate"] = date_format(order["orderCompletionDate"])
-        order["vehicleIntakeDate"] = date_format(order["vehicleIntakeDate"])
-
-        items = orders_items.get_group(order["orderNumber"]).to_dict("records")
-        order["items"] = []
-        for item in items:
-            if item["orderItemType"] == "operation":
-                order["items"].append(
-                    {
-                        "lineNumber": item["lineNumber"],
-                        "operation": {
-                            "category": item["category"],
-                            "description": item["descriptionOperation"],
-                            "hours": item["hours"],
-                            "operationCode": item["operationCode"],
-                            "standardHours": item["standardHours"],
-                        },
-                    }
-                )
-            elif item["orderItemType"] == "part":
-                order["items"].append(
-                    {
-                        "lineNumber": item["lineNumber"],
-                        "part": {
-                            "description": item["descriptionPart"],
-                            "isDamageCausal": item["isDamageCausal"],
-                            "manufacturer": item["manufacturer"],
-                            "partNumber": item["partNumber"],
-                            "quantity": item["quantity"],
-                            "serialNumber": str(item["serialNumber"]),
-                            "unit": str(item["unit"]),
-                        },
-                    }
-                )
-            elif item["orderItemType"] == "other":
-                order["items"].append(
-                    {
-                        "lineNumber": item["lineNumber"],
-                        "other": {"description": item["descriptionOther"], "type": item["type"]},
-                    }
-                )
-            else:
-                order["items"].append(
-                    {
-                        "lineNumber": item["lineNumber"],
-                        "purchaseInvoice": {
-                            "company": item["company"],
-                            "description": item["descriptionPurchase"],
-                            "invoiceCode": item["invoiceCode"],
-                            "invoiceDate": date_format(item["invoiceDate"]),
-                            "invoiceNumber": item["invoiceNumber"],
-                        },
-                    }
-                )
-
-    res = {
-        "creationDate": date_format(datetime.now()),
-        "invoices": invoices,
-        "orders": orders,
-        "timeRangeBegin": date_format(date_min),
-        "timeRangeEnd": date_format(date_max),
-    }
-
-    json.dump(res, open(export_json, "w"), indent=2)
-    return res
-
-
-def upload(oauth: OAuth2Session, data):
+def upload(cfg: MazdaConfig, oauth: OAuth2Session, data):
     headers = {
         "accept": "application/vnd.mazdaeur.dms.v5+json",
         "x-mme-organisation": cfg.dealer_number,
@@ -212,10 +75,38 @@ def upload(oauth: OAuth2Session, data):
                 fwh.write(r.content)
 
 
+def upload_files(cfg: MazdaConfig, oauth: OAuth2Session):
+    headers = {
+        "accept": "application/vnd.mazdaeur.dms.v5+json",
+        "x-mme-organisation": cfg.dealer_number,
+        "X-mazda-org": cfg.dealer_number,
+        "Content-Type": "application/json",
+    }
+    result = []
+    print("Upload Files")
+
+    for order_file in Path(base_dir + "export/Mazda").glob("*.json"):
+        data = json.load(order_file.open("r"))
+
+        r = oauth.post(cfg.domain + cfg.webservice + cfg.module, json.dumps(data), headers=headers)
+        print(f"{order_file.name} => {r.status_code}")
+        result.append(f"{order_file.name} => {r.status_code}")
+        if r.status_code in [200, 202]:
+            # OK
+            # order_file.unlink()
+            order_file.rename(base_dir + "export/Mazda/success/" + order_file.name)
+        else:
+            with open(base_dir + f"logs/{order_file.name}.log", "wb") as fwh:
+                fwh.write(r.content)
+            order_file.rename(base_dir + "export/Mazda/error/" + order_file.name)
+
+    return result
+
+
 def main():
     data = convert_csv(base_dir + "data/Workshop_Order_Report.csv", base_dir + "temp/mazda_export.json", 2021, 6)
     # data = json.load(open(base_dir + 'mazda_export.json', 'r'))
-    upload(None, data)
+    upload(cfg_debug, None, data)
 
 
 if __name__ == "__main__":

mazda_webservice.py

@@ -1,7 +1,6 @@
 import json
 import os
 from dataclasses import dataclass
-from datetime import datetime
 
 from flask import Flask, redirect, request, session
 from oauthlib.oauth2.rfc6749.errors import OAuth2Error
@@ -9,6 +8,8 @@ from requests_oauthlib import OAuth2Session
 
 import mazda_upload
 
+# from datetime import datetime
+
 
 @dataclass
 class Token:
@@ -29,7 +30,9 @@ class Token:
 
 app = Flask(__name__)
 
-cfg: mazda_upload.MazdaConfig = mazda_upload.cfg
+PROD = True
+
+cfg: mazda_upload.MazdaConfig = mazda_upload.cfg_prod if PROD else mazda_upload.cfg_debug
 base_dir = "C:/projekte/mazda/"
 
 
@@ -51,21 +54,42 @@ def get_token() -> Token:
     if session.get("oauth_token") is None:
         session["oauth_token"] = token_load()
 
-        if session["oauth_token"] is not None and session["oauth_token"]["expires_at"] < datetime.now().timestamp():
-            session["oauth_token"] = None
+        # if session["oauth_token"] is not None and session["oauth_token"]["expires_at"] < datetime.now().timestamp():
+        #     session["oauth_token"] = None
     return session["oauth_token"]
 
 
 @app.route("/")
-def demo():
-    # if token := get_token() is not None:
-    oauth = OAuth2Session(cfg.client_id, token=get_token())
+def home():
+    token = get_token()
+    access_token = token.get("access_token", "") if token else ""
+    oauth = OAuth2Session(cfg.client_id, token=token, auto_refresh_kwargs={"grant_type": "refresh_token"})
+    token_save(oauth.token)
+
+    if session["oauth_token"].get("access_token", "???") != access_token:
+        print("oh")
+
+    if not oauth.authorized:
+        token = get_token()
+        new_token = oauth.refresh_token(token.iss, token.refresh_token)
+        print("Blubb.")
+        token_save(new_token)
 
     if oauth.authorized:
         return redirect("/profile")
+
     return redirect("/login")
 
 
+@app.route("/refresh")
+def refresh():
+    oauth = OAuth2Session(cfg.client_id)
+
+    if oauth.authorized:
+        return redirect("/profile")
+    return redirect("/")
+
+
 @app.route("/login")
 def login():
     redirect_uri = request.base_url.rsplit("/", 1)[0] + "/callback"
@@ -99,18 +123,17 @@ def callback():
 
 @app.route("/profile", methods=["GET"])
 def profile():
-    oauth = OAuth2Session(cfg.client_id, token=get_token())
+    oauth = OAuth2Session(cfg.client_id, token=get_token(), auto_refresh_kwargs={"grant_type": "refresh_token"})
+
     if not oauth.authorized:
         return redirect("/")
 
-    data = mazda_upload.convert_csv(
-        base_dir + "data/Workshop_Order_Report.csv", base_dir + "temp/mazda_export.json", 2023, 12
-    )
     try:
-        mazda_upload.upload(oauth, data)
-    except OAuth2Error:
+        res = mazda_upload.upload_files(cfg, oauth)
+    except OAuth2Error as e:
+        print(e.error, e.description)
         return redirect("/")
-    return app.response_class(response=json.dumps(data, indent=2), mimetype="application/json")
+    return app.response_class(response=json.dumps(res, indent=2), mimetype="application/json")
 
 
 if __name__ == "__main__":

timestamp.py

@@ -0,0 +1,3 @@
+from datetime import datetime
+
+print(datetime.now().timestamp())