Fehlerberichte jetzt auf neuer Import-Version

fehlerbericht/config.py

@@ -1,8 +1,9 @@
 # DSN = "mysql+pymysql://gaps:Gcbs12ma@192.168.2.41/tasks"
-DSN = "mysql+pymysql://gaps:Gcbs12ma@localhost/tasks"
+# DSN = "mysql+pymysql://gaps:Gcbs12ma@localhost/tasks"
+DSN = "mysql+pymysql://gaps:Gcbs12ma@192.168.4.62/tasks"
 MAILBOX = {
     "server": "imap.1und1.de",
     "user": "status@global-cube.de",
     "password": "(6QNLU=7m3R?f2?6]JO4WH(K",
-    "folder": "Test",  # "INBOX"
+    "folder": "INBOX",  # "Test",  "INBOX"
 }

fehlerbericht/mail_import.py

@@ -32,12 +32,12 @@ class MailImport:
                 db_session.add(status)
                 try:
                     db_session.commit()
+                    imported.append(msg.uid)
                 except IntegrityError as e:
                     db_session.rollback()
                     print(e.args[0])
-                imported.append(msg.uid)
 
-            # mb.delete(imported)
+            mb.delete(imported)
 
     def get_status_message(self, msg: EmailMessage) -> StatusMeldung:
         subject = msg.subject.split(";")
@@ -45,8 +45,10 @@ class MailImport:
             print(msg.subject)
             return None
 
-        attachments = dict([(att.filename, att.payload) for att in msg.attachments])
-        fehlerbericht_json = json.loads(attachments.get("fehlerbericht.json", ""))
+        attachments = dict(
+            [(att.filename, att.payload) for att in msg.attachments if att.size > 0]
+        )
+        fehlerbericht_json = json.loads(attachments.get("fehlerbericht.json", "[]"))
         fehlerbericht_import = json.dumps(fehlerbericht_json)
 
         aufgabe = ""

fehlerbericht/prepare_logfiles.py

@@ -12,95 +12,94 @@ from sqlalchemy.orm import Session
 class PrepareLogfiles:
     def prepare_logfiles(self):
         with Session(create_engine(DSN)) as db_session:
-            query = select(
-                StatusMeldung
-            )  # .where(StatusMeldung.fehlerbericht.is_(None))
+            query = select(StatusMeldung).where(StatusMeldung.fehlerbericht.is_(None))
             for sm in db_session.scalars(query):
-                whitelist = self.get_whitelist(sm.kunde_ref.whitelist)
-                fehlerbericht = dict(
-                    [(f["Name"], f) for f in json.loads(sm.fehlerbericht_import)]
+                self.convert_logfile(sm)
+                db_session.commit()
+
+    def convert_logfile(self, sm: StatusMeldung):
+        whitelist_str = sm.kunde_ref.whitelist if sm.kunde_ref is not None else ""
+        whitelist = self.get_whitelist(whitelist_str)
+        fehlerbericht = dict(
+            [(f["Name"], f) for f in json.loads(sm.fehlerbericht_import)]
+        )
+
+        for name, report in fehlerbericht.items():
+            if ".bat" in name:
+                e = report["Errors"][0]
+                e = e.replace(
+                    "!! Bitte E-Mailadresse fuer Statusbericht angeben !!",
+                    "! Bitte E-Mailadresse fuer Statusbericht angeben !",
+                )
+                e = e.replace(
+                    "!! SMTP-Konfiguration bitte anpassen !!",
+                    "! SMTP-Konfiguration bitte anpassen !",
+                )
+                e, err_count = re.subn(r"(!![^\n]*!!)", r"<b>\1</b>", e)
+                e, err_count2 = re.subn(
+                    r"\((\w{2}\d{4})\)( [^\r]*)",
+                    r"<b><a href='/wiki/\1' target='_blank'>(\1)</a>\2</b>",
+                    e,
                 )
 
-                for name, report in fehlerbericht.items():
-                    if ".bat" in name:
-                        e = report["Errors"][0]
-                        e = e.replace(
-                            "!! Bitte E-Mailadresse fuer Statusbericht angeben !!",
-                            "! Bitte E-Mailadresse fuer Statusbericht angeben !",
-                        )
-                        e = e.replace(
-                            "!! SMTP-Konfiguration bitte anpassen !!",
-                            "! SMTP-Konfiguration bitte anpassen !",
-                        )
-                        e, err_count = re.subn(r"(!![^\n]*!!)", r"<b>\1</b>", e)
-                        e, err_count2 = re.subn(
-                            r"\((\w{2}\d{4})\)( [^\r]*)",
-                            r"<b><a href='/wiki/\1' target='_blank'>(\1)</a>\2</b>",
-                            e,
-                        )
-
-                        e, _ = re.subn(
-                            r"Error \= \[Micro.* NULL.*\.",
-                            "",
-                            e,
-                        )
-
-                        err_count3 = 0
-                        e, err_count3 = re.subn(
-                            r"(Error \= \[Micro[^\r]*)",
-                            r"<b>\1</b>",
-                            e,
-                        )
-
-                        if sum([err_count, err_count2, err_count3]) > 0:
-                            report["ErrorLevel"] = 2
-                            report["Errors"] = [e] + re.findall(r"<b>[^\n]*</b>", e)
-                            sm.anzahl += 1
-                    elif ".dtsx" in name:
-                        e = re.sub(r"\r\n", "\n", report["Errors"][0])
-                        e = re.sub(r"(\w*)ende\n", "§", e)
-                        e = re.sub(r"Copyright ", "§", e)
-                        loglevel_warn = [b for b in e.split("§") if "Status:" not in b]
-                        loglevel_error = [b for b in loglevel_warn if "Fehler:" in b]
-                        loglevel_combined = [
-                            "<b>" + b + "</b>" if "Fehler:" in b else b
-                            for b in loglevel_warn
-                        ]
-                        loglevel_combined[-1] = loglevel_combined[-1].replace(
-                            "DTSER_FAILURE", "<b>DTSER_FAILURE</b>"
-                        )
-                        if (
-                            len(loglevel_error) > 0
-                            or "DTSER_SUCCESS" not in loglevel_combined[-1]
-                        ):
-                            report["ErrorLevel"] = 2
-                            sm.anzahl += 1
-                        report["Errors"][0] = "\n".join(loglevel_combined)
-                    elif len(report["Errors"]) > 0 and report["Type"] in [
-                        "Portal",
-                        "Versand",
-                    ]:
-                        report["Errors2"] = [
-                            e
-                            for e in report["Errors"]
-                            if self.in_whitelist(e, whitelist)
-                        ]
-                        report["Errors"] = [
-                            e for e in report["Errors"] if e not in report["Errors2"]
-                        ]
-                        if len(report["Errors"]) == 0:
-                            report["ErrorLevel"] = 3
-                            sm.anzahl -= 1
-
-                sm.fehlerbericht = json.dumps(list(fehlerbericht.values()), indent=2)
-                self.export_log_files(sm, fehlerbericht)
-                # batch_log = [f for f in fehlerbericht if ".bat" in f["Name"]].pop(0)
-                systeminfo_log = SysteminfoLog(
-                    fehlerbericht.get("system.info", {"Errors": [""]})
+                e, _ = re.subn(
+                    r"Error \= \[Micro.* NULL.*\.",
+                    "",
+                    e,
                 )
-                # if systeminfo_log.ip_address == "0.0.0.0":
-                print(sm.kunde)
-                db_session.commit()
+
+                err_count3 = 0
+                e, err_count3 = re.subn(
+                    r"(Error \= \[Micro[^\r]*)",
+                    r"<b>\1</b>",
+                    e,
+                )
+
+                if sum([err_count, err_count2, err_count3]) > 0:
+                    report["ErrorLevel"] = 2
+                    report["Errors"] = [e] + re.findall(r"<b>[^\n]*</b>", e)
+                    sm.anzahl += 1
+            elif ".dtsx" in name:
+                e = re.sub(r"\r\n", "\n", report["Errors"][0])
+                e = re.sub(r"(\w*)ende\n", "§", e)
+                e = re.sub(r"Copyright ", "§", e)
+                loglevel_warn = [b for b in e.split("§") if "Status:" not in b]
+                loglevel_error = [b for b in loglevel_warn if "Fehler:" in b]
+                loglevel_combined = [
+                    "<b>" + b + "</b>" if "Fehler:" in b else b for b in loglevel_warn
+                ]
+                loglevel_combined[-1] = loglevel_combined[-1].replace(
+                    "DTSER_FAILURE", "<b>DTSER_FAILURE</b>"
+                )
+                if (
+                    len(loglevel_error) > 0
+                    or "DTSER_SUCCESS" not in loglevel_combined[-1]
+                ):
+                    report["ErrorLevel"] = 2
+                    sm.anzahl += 1
+                report["Errors"][0] = "\n".join(loglevel_combined)
+            elif len(report["Errors"]) > 0 and report["Type"] in [
+                "Portal",
+                "Versand",
+            ]:
+                report["Errors2"] = [
+                    e for e in report["Errors"] if self.in_whitelist(e, whitelist)
+                ]
+                report["Errors"] = [
+                    e for e in report["Errors"] if e not in report["Errors2"]
+                ]
+                if len(report["Errors"]) == 0:
+                    report["ErrorLevel"] = 3
+                    sm.anzahl -= 1
+
+        sm.fehlerbericht = json.dumps(list(fehlerbericht.values()), indent=2)
+        self.export_log_files(sm, fehlerbericht)
+        # batch_log = [f for f in fehlerbericht if ".bat" in f["Name"]].pop(0)
+        systeminfo_log = SysteminfoLog(
+            fehlerbericht.get("system.info", {"Errors": [""]})
+        )
+        # if systeminfo_log.ip_address == "0.0.0.0":
+        print(sm.kunde)
 
     def in_whitelist(self, e, whitelist):
         return any(
@@ -133,6 +132,17 @@ class PrepareLogfiles:
         return whitelist
 
 
-if __name__ == "__main__":
+def main():
     mi = PrepareLogfiles()
     mi.prepare_logfiles()
+
+
+if __name__ == "__main__":
+    main()
+
+    # import cProfile
+
+    # cProfile.run(
+    #     "main()",
+    #     "fehlerbericht/prepare_logfiles.prof",
+    # )

requirements.txt

@@ -4,6 +4,7 @@ colorama
 cryptography
 Flask
 Flask_Cors
+imap_tools
 Jinja2
 numpy
 oauthlib
@@ -13,6 +14,7 @@ pdfplumber
 plac
 psutil
 pygame
+pymysql
 pyodbc
 python_crontab
 python_dateutil