C11 Version dynamisch

c11.py

@@ -18,9 +18,9 @@ cfg = config.Config()
 
 
 @app.command()
-def export(folder=""):
+def export(folder="", overwrite="0"):
     exp = cognos11.c11_export(cfg)
-    exp.export_folder(folder)
+    exp.export_folder(folder, overwrite=overwrite != "0")
 
 
 @app.command()
@@ -32,6 +32,8 @@ def reportoutput(folder=""):
     pdf_test.missing_data(cfg.cognos11.reportoutput_dir + "/" + folder)
     merge_group = exp.get_merge_group(req_plan)
     pdf_merge.pdf_merge_files(merge_group)
+    exp.template(req_plan, merge_group)
+    cognos11.get_converted_jobs(cfg)
     # pdf_merge.merge_reports_in_folder(self.cfg, folder2)
 
 
@@ -56,7 +58,10 @@ def create(path: str):
 @app.command()
 def template():
     exp = cognos11.c11_export(cfg)
-    exp.mail_template("")
+    cognos11.get_converted_jobs(cfg)
+    req_plan = exp.get_folder_pdf_request_plan("")
+    merge_group = exp.get_merge_group(req_plan)
+    exp.template(req_plan, merge_group)
 
 
 @app.command()
@@ -76,3 +81,4 @@ def mdlconvert(mdl_file):
 
 if __name__ == "__main__":
     app()
+    # reportoutput()

cognos11/__init__.py

@@ -1,3 +1,4 @@
 from cognos11.c11_export import c11_export
 from cognos11.c11_create import c11_create
 from cognos11.c11_api import c11_api
+from cognos11.c11_jobs import get_converted_jobs

cognos11/c11_api.py

@@ -21,6 +21,7 @@ class c11_api:
     reports = []
     folders = []
     jobs = []
+    server_version = "202004"
 
     def __init__(self, cfg: config.Config):
         self.cfg = cfg
@@ -78,8 +79,17 @@ class c11_api:
 
         self.caf = r.json()["cafContextId"]
         self.cam = self.generate_token(r.cookies["usersessionid"])
+
+        self.server_version = self.get_server_version()
         return self
 
+    def get_server_version(self):
+        r = self.session.get(self.webservice + "pat/rsstartupblock_3.js")
+        res = re.search(r"\=\"(\d{6})\";", r.text)
+        if res:
+            return res.group(1)
+        return "202004"
+
     def get_folders(self):
         if len(self.folders) == 0:
             self.folders.append({"id": "_dot_public_folders", "name": "Team Content"})
@@ -150,9 +160,7 @@ class c11_api:
                     report["parameters"] = {}
                     params = [p for p in f["parameters"] if len(p["value"]) > 0]
                     for p in params:
-                        report["parameters"][p["name"]] = dict(
-                            [(v["use"], v["display"]) for v in p["value"]]
-                        )
+                        report["parameters"][p["name"]] = dict([(v["use"], v["display"]) for v in p["value"]])
                     # report["details"] = self.get_report_details(report["id"])
                     # print(json.dumps(f, indent=2))
                 self.reports.append(report)
@@ -268,7 +276,7 @@ class c11_api:
             "X-XSRF-TOKEN": self.headers["X-XSRF-TOKEN"],
             "X-RsCMStoreID": report["id"],
             "X-UseRsConsumerMode": "true",
-            "SOAPAction": "http://www.ibm.com/xmlns/prod/cognos/reportService/202004/",
+            "SOAPAction": f"http://www.ibm.com/xmlns/prod/cognos/reportService/{self.server_version}/",
         }
         soap = self.templates["get_report"].render(
             {
@@ -282,9 +290,7 @@ class c11_api:
             }
         )
 
-        r = self.session.post(
-            self.webservice + "v1/reports", data=soap, headers=headers
-        )
+        r = self.session.post(self.webservice + "v1/reports", data=soap, headers=headers)
         if r.status_code == 500:
             bs = BeautifulSoup(r.text, "xml")
             report["error"] = bs.find_all("messageString")[0].string
@@ -318,27 +324,14 @@ class c11_api:
 
         for sv in bs.find_all("selectvalue"):
             k = sv["parameter"]
-            v = dict(
-                [
-                    (opt["usevalue"], opt.get("displayvalue", ""))
-                    for opt in sv.find_all("selectoption")
-                ]
-            )
+            v = dict([(opt["usevalue"], opt.get("displayvalue", "")) for opt in sv.find_all("selectoption")])
             meta["optional"][k] = v
 
         for sv in bs.find_all("selectdate"):
             k = sv["parameter"]
-            v = dict(
-                [
-                    (opt["usevalue"], opt.get("displayvalue", ""))
-                    for opt in sv.find_all("selectoption")
-                ]
-            )
+            v = dict([(opt["usevalue"], opt.get("displayvalue", "")) for opt in sv.find_all("selectoption")])
             meta["optional"][k] = v
-        filename = (
-            self.cfg.cognos11.config_dir
-            + f"/params/{report['path']}/{report['name']}.json"
-        )
+        filename = self.cfg.cognos11.config_dir + f"/params/{report['path']}/{report['name']}.json"
         os.makedirs(os.path.dirname(filename), exist_ok=True)
         json.dump(meta, open(filename, "w"), indent=2)
         report["meta"] = meta
@@ -347,9 +340,7 @@ class c11_api:
 
     def get_report_filename(self, report):
         path = report["path"].replace("Team Content/ReportOutput", "")
-        report[
-            "filename"
-        ] = f"{self.cfg.cognos11.reportoutput_dir}/{path}/{report['name']}.pdf"
+        report["filename"] = f"{self.cfg.cognos11.reportoutput_dir}/{path}/{report['name']}.pdf"
         report["format"] = "PDF"
         if report["name"][-5:].lower() == ".xlsx":
             report["format"] = "spreadsheetML"
@@ -357,18 +348,14 @@ class c11_api:
 
         report["params"] = list(re.findall(r"\[([^\]]+)\]", report["filename"]))
         for i, p in enumerate(report["params"]):
-            report["filename"] = report["filename"].replace(
-                "[" + p + "]", "{" + str(i) + "}"
-            )
+            report["filename"] = report["filename"].replace("[" + p + "]", "{" + str(i) + "}")
         return report
 
     def request_unstubbed(self, report_id):
         report = self.get_report(report_id)
         if "spec" not in report:
             return ""
-        payload = json.dumps(
-            {"reportspec_stubbed": report["spec"], "storeid": report["id"]}
-        )
+        payload = json.dumps({"reportspec_stubbed": report["spec"], "storeid": report["id"]})
 
         headers = {
             "Content-Type": "application/json; charset=UTF-8",
@@ -378,9 +365,7 @@ class c11_api:
             "cafContextId": self.caf,
         }
 
-        r = self.session.post(
-            self.webservice + "v1/reports/unstubreport", data=payload, headers=headers
-        )
+        r = self.session.post(self.webservice + "v1/reports/unstubreport", data=payload, headers=headers)
         if r.status_code != 200:
             logging.error(f"{report['path']}/{report['name']}")
             logging.error(r.text)
@@ -414,7 +399,7 @@ class c11_api:
             "Content-Type": "text/xml; charset=UTF-8",
             "X-XSRF-TOKEN": self.headers["X-XSRF-TOKEN"],
             "X-UseRsConsumerMode": "true",
-            "SOAPAction": "http://www.ibm.com/xmlns/prod/cognos/reportService/202004/",
+            "SOAPAction": f"http://www.ibm.com/xmlns/prod/cognos/reportService/{self.server_version}/",
         }
         if report_id is not None:
             res["X-RsCMStoreID"] = report_id
@@ -439,9 +424,7 @@ class c11_api:
             )
             .encode("utf-8")
         )
-        r = self.session.post(
-            self.webservice + "v1/reports", data=soap, headers=headers
-        )
+        r = self.session.post(self.webservice + "v1/reports", data=soap, headers=headers)
 
         bs = BeautifulSoup(r.text, "xml")
         if r.status_code == 200:
@@ -461,22 +444,14 @@ class c11_api:
         # self.session.get(self.webservice + 'v1/reports/startupconfig?keys=supportedContentLocales,supportedCurrencies,
         # supportedFonts,metadataInformationURI,glossaryURI&locale=de')
         headers = self.get_report_headers()
-        soap = (
-            self.templates["get_package"]
-            .render({"caf": self.caf, "cam": self.cam})
-            .encode("utf-8")
-        )
-        r = self.session.post(
-            self.webservice + "v1/reports", data=soap, headers=headers
-        )
+        soap = self.templates["get_package"].render({"caf": self.caf, "cam": self.cam}).encode("utf-8")
+        r = self.session.post(self.webservice + "v1/reports", data=soap, headers=headers)
         open("package.xml", "wb").write(r.content)
 
         search_path = self.request_search_path(folder_id)
         report_name = os.path.basename(fullpath)
         unstubbed = open(fullpath, "rb").read()
-        headers[
-            "SOAPAction"
-        ] = "http://www.ibm.com/xmlns/prod/cognos/reportService/202004/.session"
+        headers["SOAPAction"] = f"http://www.ibm.com/xmlns/prod/cognos/reportService/{self.server_version}/.session"
         headers["Referer"] = "http://localhost:9300/bi/pat/rsapp.htm"
         # headers['caf'] = self.caf
 
@@ -493,9 +468,7 @@ class c11_api:
             )
             .encode("utf-8")
         )
-        r = self.session.post(
-            self.webservice + "v1/reports", data=soap, headers=headers
-        )
+        r = self.session.post(self.webservice + "v1/reports", data=soap, headers=headers)
         open("request_create.xml", "wb").write(r.request.body)
         print(r.status_code)
         print(r.text)
@@ -515,9 +488,7 @@ class c11_api:
                 "unstubbed": unstubbed,
             }
         )  # .encode("utf-8")
-        r = self.session.post(
-            self.webservice + "v1/reports", data=soap, headers=headers
-        )
+        r = self.session.post(self.webservice + "v1/reports", data=soap, headers=headers)
         # open('request_update.xml', 'wb').write(r.request.body)
         print(r.status_code)
         print(r.text)
@@ -537,9 +508,7 @@ class c11_api:
             return folder_id
 
     def request_search_path(self, id):
-        res = self.session.get(
-            f"{self.webservice}v1/objects/{id}?fields=searchPath", headers=self.headers
-        )
+        res = self.session.get(f"{self.webservice}v1/objects/{id}?fields=searchPath", headers=self.headers)
         return res.json()["data"][0]["searchPath"]
 
 

cognos11/c11_export.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from pathlib import Path
 from cognos11.c11_api import c11_api
 import config
@@ -48,29 +49,42 @@ class c11_export:
             folder = "Team Content/ReportOutput/" + folder
         return folder
 
-    def export_folder(self, folder="") -> None:
+    def export_folder(self, folder="", overwrite=False) -> None:
         folder = self.get_folder(folder, "XML")
         reports = self.api.get_reports_in_folder(folder, True)
         for r in reports:
-            self.export_unstubbed(r["id"])
+            if self.is_obsolete_or_ignored(r):
+                continue
+            self.export_unstubbed(r["id"], overwrite)
 
-    def export_unstubbed(self, report_id):
+    @staticmethod
+    def is_obsolete_or_ignored(report):
+        if report["type"] != "report":
+            return True
+        path: str = report["path"]
+        if "_/" in path or path.endswith("_"):
+            return True
+        for prefix in ["Team Content/Templates", "Team Content/Beispiele", "Team Content/Kalender"]:
+            if path.startswith(prefix):
+                return True
+        name: str = report["name"]
+        if name.endswith("_") or "Kopie" in name:
+            return True
+        return False
+
+    def export_unstubbed(self, report_id, overwrite=False):
         report = self.api.get_report(report_id)
         if "error" in report:
             return
 
         params = self.get_params(report, {})
-        request = ReportRequest(
-            report["id"], params, report["filename"], report["format"]
-        )
+        request = ReportRequest(report["id"], params, report["filename"], report["format"])
         # test if execution of report is possible
         self.request_and_save_file(request, save=False)
 
-        filename = (
-            f"{self.cfg.cognos11.specs_dir}/{report['path']}/{report['name']}.xml"
-        )
+        filename = f"{self.cfg.cognos11.specs_dir}/{report['path']}/{report['name']}.xml"
         modified = datetime.fromisoformat(report["modified"]).timestamp()
-        if Path(filename).exists() and Path(filename).stat().st_mtime > modified:
+        if not overwrite and Path(filename).exists() and Path(filename).stat().st_mtime > modified:
             return
 
         unstubbed_report = self.api.request_unstubbed(report_id)
@@ -105,39 +119,29 @@ class c11_export:
         if len(report["params"]) == 1:
             filename = report["filename"].format("_Summe")
             params = self.get_params(report, {})
-            result.append(
-                ReportRequest(report["id"], params, filename, report["format"])
-            )
+            result.append(ReportRequest(report["id"], params, filename, report["format"]))
 
             key1 = report["params"][0]
             for k1, v1 in report["meta"]["optional"][key1].items():
                 filename = report["filename"].format(v1)
                 params = self.get_params(report, {key1: {k1: v1}})
-                result.append(
-                    ReportRequest(report["id"], params, filename, report["format"])
-                )
+                result.append(ReportRequest(report["id"], params, filename, report["format"]))
             return result
         if len(report["params"]) == 2:
             filename = report["filename"].format("_Summe", "").replace("_.", ".")
             params = self.get_params(report, {})
-            result.append(
-                ReportRequest(report["id"], params, filename, report["format"])
-            )
+            result.append(ReportRequest(report["id"], params, filename, report["format"]))
 
             key1, key2 = report["params"]
             for k1, v1 in report["meta"]["optional"][key1].items():
                 filename = report["filename"].format(v1, "_Summe")
                 params = self.get_params(report, {key1: {k1: v1}})
-                result.append(
-                    ReportRequest(report["id"], params, filename, report["format"])
-                )
+                result.append(ReportRequest(report["id"], params, filename, report["format"]))
 
                 for k2, v2 in report["meta"]["optional"][key2].items():
                     filename = report["filename"].format(v1, v2)
                     params = self.get_params(report, {key1: {k1: v1}, key2: {k2: v2}})
-                    result.append(
-                        ReportRequest(report["id"], params, filename, report["format"])
-                    )
+                    result.append(ReportRequest(report["id"], params, filename, report["format"]))
             return result
 
     def get_params(self, report, optional_params):
@@ -145,9 +149,10 @@ class c11_export:
         params.update(optional_params)
         return params
 
-    def execute_request_plan(self, req_plan):
+    def execute_request_plan(self, req_plan: list[list[ReportRequest]]):
         for req_group in req_plan:
             for report_req in req_group:
+                print(Path(report_req.filename).relative_to(self.cfg.cognos11.reportoutput_dir))
                 self.request_and_save_file(report_req)
 
     def request_and_save_file(self, report_request: ReportRequest, save=True):
@@ -167,12 +172,12 @@ class c11_export:
         with open(report_request.filename, "wb") as f:
             f.write(content)
 
-    def get_merge_group(self, req_plan):
+    def get_merge_group(self, req_plan: list[list[ReportRequest]]):
         res = {}
         for req_group in req_plan:
-            if len(req_group) == 1:
+            if len(req_group) <= 1:
                 continue
-            files = [r.filename for r in req_group]
+            files = [r.filename for r in req_group if r.report_format == "PDF"]
             filename = files[0].replace("_Summe", "_Summe_und_Schichten")
             res[filename] = files
 
@@ -182,33 +187,46 @@ class c11_export:
         return res
 
     def export_errors(self):
-        reports = self.api.get_reports_in_folder(
-            "Team Content", recursive=True, specs=True
-        )
+        reports = self.api.get_reports_in_folder("Team Content", recursive=True, specs=True)
         errors = [r for r in reports if "error" in r]
         filename = self.cfg.cognos11.logs_dir + "/c11_report_errors.json"
         json.dump(errors, open(filename, "w"), indent=2)
 
+    def template(self, req_plan: list[list[ReportRequest]], merge_group: dict[str, list[str]]):
+        mail_groups = defaultdict(list)
+        for report in merge_group.keys():
+            filename = str(Path(report).relative_to(self.cfg.cognos11.reportoutput_dir))
+            folder = Path(filename).parent.name
+            mail_groups[folder].append(filename)
+
+        for req_group in req_plan:
+            for report in req_group:
+                filename = str(Path(report.filename).relative_to(self.cfg.cognos11.reportoutput_dir))
+                folder = Path(filename).parent.name
+                mail_groups[folder].append(filename)
+        for group, file_list in mail_groups.items():
+            with open(
+                self.cfg.cognos11.config_dir + f"/{group}.template.csv",
+                "w",
+                encoding="latin-1",
+            ) as fwh:
+                fwh.write("Datei;Empfaenger\n")
+                for filename in file_list:
+                    fwh.write(filename + ";\n")
+        return True
+
     def mail_template(self, folder: str):
         current_dir = Path(self.cfg.cognos11.reportoutput_dir + "\\" + folder)
 
-        subfolders = set(
-            [f.name for f in current_dir.glob("*") if f.is_dir()]
-        ).difference([".", "..", "leer"])
+        subfolders = set([f.name for f in current_dir.glob("*") if f.is_dir()]).difference([".", "..", "leer"])
 
         folder2 = folder + "\\" if folder != "" else ""
 
-        export_files = [
-            f"{folder2}{f.name};"
-            for f in current_dir.glob("*.*")
-            if f.suffix in [".pdf", ".xls"]
-        ]
+        export_files = [f"{folder2}{f.name};" for f in current_dir.glob("*.*") if f.suffix in [".pdf", ".xls"]]
 
         if len(export_files) > 0:
             folder2 = folder.replace("\\", "_") if folder != "" else "Versand"
-            template_file = Path(
-                self.cfg.cognos11.config_dir + "\\" + folder2 + ".template.csv"
-            )
+            template_file = Path(self.cfg.cognos11.config_dir + "\\" + folder2 + ".template.csv")
             print(template_file)
             with template_file.open("w") as fwh:
                 fwh.write("Report;Empfaenger\n")

cognos11/c11_jobs.py

@@ -1,32 +1,72 @@
 import json
+import csv
+import config
 
 
-with open("data/reports.json", "r") as frh:
-    reports = json.load(frh)
+def get_converted_jobs(cfg: config.Config):
+    with open(cfg.cognos11.reports_file, "r") as frh:
+        reports = json.load(frh)
 
-report_id_to_name = dict([(r["id"], r["name"]) for r in reports])
+    report_id_to_name = dict([(r["id"], r["name"]) for r in reports])
 
-with open("data/jobs.json", "r") as frh:
-    data = json.load(frh)
+    with open(cfg.cognos11.jobs_file, "r") as frh:
+        data = json.load(frh)
 
-for job in data:
-    print(job["name"])
+    result = []
+    for job in data:
+        print(job["name"])
 
-    for step in job["details"]["steps"]:
-        to_address = [
-            opt["value"] for opt in step["options"] if opt["name"] == "toAddress"
-        ]
-        parameters = {}
-        if step["parameters"]:
-            parameters = dict(
-                [
-                    (opt["name"], opt["value"])
-                    for opt in step["parameters"]
-                    if len(opt["value"]) > 0
+        for step in job["details"]["steps"]:
+            res = {}
+            res["job_name"] = job["name"]
+            res["to_address"] = ",".join(
+                [opt["value"] for opt in step["options"] if opt["name"] == "toAddress"][
+                    0
                 ]
             )
+            res["params"] = {}
+            if step["parameters"]:
+                res["params"] = dict(
+                    [
+                        (opt["name"], opt["value"][0]["display"])
+                        for opt in step["parameters"]
+                        if len(opt["value"]) > 0
+                    ]
+                )
 
-        report_name = report_id_to_name[step["report_id"]]
-        print("  " + report_name)
-        print("    " + ",\n    ".join(to_address[0]))
-        # print("    ", parameters)
+            res["report_name"] = report_id_to_name[step["report_id"]]
+            res["subject"] = [
+                opt["value"] for opt in step["options"] if opt["name"] == "subject"
+            ][0]
+            print("  " + res["report_name"])
+            # print("    " + ",\n    ".join(res["to_address"]))
+            # print("    ", res["params"])
+            result.append(res)
+
+    if len(result) == 0:
+        return
+
+    with open(
+        cfg.cognos11.config_dir + "/c11_jobs.csv", "w", encoding="latin-1", newline=""
+    ) as fwh:
+        csv_writer = csv.writer(fwh, delimiter=";")
+        csv_writer.writerow(result[0].keys())
+        for row in result:
+            csv_writer.writerow(row.values())
+
+    with open(
+        cfg.cognos11.config_dir + "/converted.template.csv",
+        "w",
+        encoding="latin-1",
+        newline="",
+    ) as fwh:
+        csv_writer = csv.writer(fwh, delimiter=";")
+        csv_writer.writerow(["Report", "Empfaenger"])
+        for row in result:
+            suffix = (
+                "_Summe"
+                if len(row["params"].values()) == 0
+                else list(row["params"].values())[0]
+            )
+            filename = f'{row["job_name"]}\\{row["report_name"]}_{suffix}.pdf'
+            csv_writer.writerow([filename, row["to_address"].lower()])

dist/c11_reportoutput.bat

@@ -0,0 +1,3 @@
+@call "%~dp0config.bat" 0 > nul
+
+gctools.exe c11 reportoutput --folder "%~1"

gctools.py

@@ -0,0 +1,13 @@
+import typer
+import c11
+import db
+import xls
+
+app = typer.Typer()
+app.add_typer(c11.app, name="c11")
+app.add_typer(db.app, name="db")
+app.add_typer(xls.app, name="excel")
+
+
+if __name__ == "__main__":
+    app()

pdf/pdf_merge.py

@@ -27,7 +27,7 @@ def pdf_merge(base_dir: str, report_name, config=None):
                 merger.write(target)
 
 
-def pdf_merge_files(file_dict):
+def pdf_merge_files(file_dict: dict[str, list[str]]):
     max_age = datetime.now().timestamp() - 12 * 60 * 60
 
     for target, file_list in file_dict.items():