Cognos 11 workflow

c11.py

@@ -1,5 +1,6 @@
 import config
 import cognos11
+from pdf import pdf_merge, pdf_test
 import plac
 from enum import Enum
 
@@ -12,24 +13,28 @@ class ExportFormat(Enum):
 class C11:
     commands = ['export', 'reportoutput', 'errors', 'create']
 
+    def __init__(self):
+        self.cfg = config.Config()
+
     def export(self, folder='', format='XML'):
-        cfg = config.Config()
-        exp = cognos11.c11_export(cfg)
+        exp = cognos11.c11_export(self.cfg)
         exp.export_folder(folder, format)
 
     def reportoutput(self, folder=''):
-        self.export(folder, 'PDF')
-        
+        exp = cognos11.c11_export(self.cfg)
+        folder2 = exp.get_folder(folder)
+        # exp.export_folder(folder, 'PDF')
+        pdf_test.missing_data(self.cfg.cognos11.reportoutput_dir + '/' + folder)
+        pdf_merge.merge_reports_in_folder(self.cfg, folder2)
 
     def errors(self):
-        cfg = config.Config()
-        exp = cognos11.c11_export(cfg)
+        exp = cognos11.c11_export(self.cfg)
         exp.export_errors()
 
     def create(self, path: str):
-        cfg = config.Config()
-        cognos11.c11_create(cfg).create_path(path)
+        cognos11.c11_create(self.cfg).create_path(path)
 
 
 if __name__ == '__main__':
-    plac.Interpreter.call(C11)
+    # plac.Interpreter.call(C11)
+    C11().reportoutput()

cognos11/__init__.py

@@ -1,2 +1,3 @@
 from cognos11.c11_export import c11_export
 from cognos11.c11_create import c11_create
+from cognos11.c11_api import c11_api

cognos11/c11_api.py

@@ -6,7 +6,8 @@ import jinja2
 import json
 import re
 from bs4 import BeautifulSoup
-from .xml_prettify import prettify_xml
+import config
+from cognos11.xml_prettify import prettify_xml
 import logging
 
 
@@ -14,8 +15,6 @@ class c11_api:
     webservice = ""
     templates_dir = ""
     # templates_dir = "C:/GlobalCube/Tasks/gctools/templates"
-    export_dir = "C:/GlobalCube/ReportOutput"
-    log_dir = "C:/GlobalCube/Tasks/gctools/logs"
     headers = {}
     caf = ""
     cam = ""
@@ -23,8 +22,9 @@ class c11_api:
     folders = []
     jobs = []
 
-    def __init__(self, webservice="http://localhost:9300/bi/"):
-        self.webservice = webservice
+    def __init__(self, cfg: config.Config):
+        self.cfg = cfg
+        self.webservice = cfg.cognos11.webservice
 
         self.templates_dir = os.path.dirname(__file__) + '/templates'
         self._env = jinja2.Environment(
@@ -76,8 +76,26 @@ class c11_api:
         if len(self.folders) == 0:
             self.folders.append({'id': '_dot_public_folders', 'name': 'Team Content'})
             self.load_folder_list()
+            self.save_config()
         return self.folders
 
+    def save_config(self):
+        os.makedirs(self.cfg.cognos11.config_dir, exist_ok=True)
+        with open(self.cfg.cognos11.folders_file, 'w') as fwh:
+            json.dump(self.folders, fwh, indent=2)
+        with open(self.cfg.cognos11.reports_file, 'w') as fwh:
+            json.dump(self.reports, fwh, indent=2)
+        with open(self.cfg.cognos11.jobs_file, 'w') as fwh:
+            json.dump(self.jobs, fwh, indent=2)
+
+    def load_config_from_files(self):
+        with open(self.cfg.cognos11.folders_file, 'r') as frh:
+            self.folders = json.load(frh)
+        with open(self.cfg.cognos11.reports_file, 'r') as frh:
+            self.reports = json.load(frh)
+        with open(self.cfg.cognos11.jobs_file, 'w') as frh:
+            self.jobs = json.load(frh)
+
     def load_folder_list(self, folder_id='_dot_public_folders', prefix='Team Content'):
         res = self.session.get(f"{self.webservice}v1/objects/{folder_id}/items", headers=self.headers)
         folder_list = sorted(res.json()['data'], key=lambda x: x['defaultName'])
@@ -197,13 +215,13 @@ class c11_api:
             req = 'required' if sv['required'] == 'true' else 'optional'
             v = dict([(opt['usevalue'], opt.get('displayvalue', '')) for opt in sv.find_all('selectoption')])
             meta[req][k] = v
-        filename = self.log_dir + f"/config/{report['path']}/{report['name']}.json"
+        filename = self.cfg.xml_dir + f"/{report['path']}/{report['name']}.json"
         os.makedirs(os.path.dirname(filename), exist_ok=True)
         json.dump(meta, open(filename, 'w'), indent=2)
         report['meta'] = meta
         report['spec'] = parts[2].text
         path = report['path'].replace('Team Content/ReportOutput', '')
-        report['filename'] = f"{self.export_dir}/{path}/{report['name']}.pdf"
+        report['filename'] = f"{self.cfg.cognos11.reportoutput_dir}/{path}/{report['name']}.pdf"
 
         report['params'] = list(re.findall(r'\[([^\]]+)\]', report['filename']))
         for i, p in enumerate(report['params']):
@@ -284,8 +302,10 @@ class c11_api:
         return r.status_code, error
 
     def create_report(self, folder_id, fullpath):
-        # self.session.get(self.webservice + 'v1/reports/templates?path=%2Fcontent%2Ffolder%5B%40name%3D%27Templates%27%5D/*[@objectClass=%27interactiveReport%27%20or%20@objectClass=%27report%27%20or%20@objectClass=%27reportTemplate%27]&maxResults=100&locale=de')
-        # self.session.get(self.webservice + 'v1/reports/startupconfig?keys=supportedContentLocales,supportedCurrencies,supportedFonts,metadataInformationURI,glossaryURI&locale=de')
+        # self.session.get(self.webservice + 'v1/reports/templates?path=%2Fcontent%2Ffolder%5B%40name%3D%27Templates%27%5D/
+        # *[@objectClass=%27interactiveReport%27%20or%20@objectClass=%27report%27%20or%20@objectClass=%27reportTemplate%27]&maxResults=100&locale=de')
+        # self.session.get(self.webservice + 'v1/reports/startupconfig?keys=supportedContentLocales,supportedCurrencies,
+        # supportedFonts,metadataInformationURI,glossaryURI&locale=de')
         headers = self.get_report_headers()
         soap = self.templates['get_package'].render(
             {"caf": self.caf, "cam": self.cam}

cognos11/c11_export.py

@@ -1,25 +1,26 @@
+from cognos11.c11_api import c11_api
+import config
 import json
 import logging
 import os
-from .c11_api import c11_api
 from datetime import datetime
 
 
 class c11_export:
     api: c11_api
-    config = None
-    log_dir = "C:/GlobalCube/Tasks/logs/c11"
-    spec_dir = "C:/GlobalCube/System/OPTIMA/Report"
+    cfg: config.Config = None
 
-    def __init__(self, cfg, api=None):
-        self.config = cfg
+    def __init__(self, cfg=None, api=None):
+        self.cfg = cfg
+        if cfg is None:
+            self.cfg = config.Config()
         self.api = api
         if api is None:
-            self.api = c11_api().login()
+            self.api = c11_api(cfg).login()
 
         now = datetime.now().strftime('%Y%m%d_%H%M%S')
-        prot_file = f"{self.log_dir}/error_{now}.log"
-        os.makedirs(self.log_dir, exist_ok=True)
+        prot_file = f"{self.cfg.cognos11.logs_dir}/error_{now}.log"
+        os.makedirs(self.cfg.cognos11.logs_dir, exist_ok=True)
         logging.basicConfig(
             filename=prot_file,
             filemode='w',
@@ -28,12 +29,15 @@ class c11_export:
             force=True
         )
 
-    def export_folder(self, folder='', format='PDF') -> None:
+    def get_folder(self, folder, format='PDF'):
         if folder == '':
             folder = 'Team Content' if format == 'XML' else 'Team Content/ReportOutput'
         elif not folder.startswith('Team Content'):
             folder = 'Team Content/ReportOutput/' + folder
+        return folder
 
+    def export_folder(self, folder='', format='PDF') -> None:
+        folder = self.get_folder(folder, format)
         reports = self.api.get_reports_in_folder(folder, True)
         for r in reports:
             print(r['name'])
@@ -46,7 +50,7 @@ class c11_export:
         report = self.api.get_report(report_id)
         unstubbed_report = self.api.request_unstubbed(report_id)
         if unstubbed_report:
-            filename = f"{self.spec_dir}/{report['path']}/{report['name']}.xml"
+            filename = f"{self.cfg.cognos11.specs_dir}/{report['path']}/{report['name']}.xml"
             os.makedirs(os.path.dirname(filename), exist_ok=True)
             with open(filename, "w") as f:
                 f.write(unstubbed_report)
@@ -109,7 +113,7 @@ class c11_export:
     def export_errors(self):
         reports = self.api.get_reports_in_folder('Team Content', recursive=True, specs=True)
         errors = [r for r in reports if 'error' in r]
-        filename = 'C:/GlobalCube/Tasks/logs/c11_report_errors.json'
+        filename = self.cfg.cognos11.logs_dir + '/c11_report_errors.json'
         json.dump(errors, open(filename, 'w'), indent=2)
 
 

config/config.py

@@ -24,6 +24,18 @@ class Cognos7Config:
     prot_dir: str
 
 
+@dataclass
+class Cognos11Config:
+    logs_dir: str
+    reportoutput_dir: str
+    specs_dir: str
+    webservice: str
+    config_dir: str
+    folders_file: str
+    reports_file: str
+    jobs_file: str
+
+
 def joinpath(path, *other):
     return str(Path(path).joinpath(*other)).lower()
 
@@ -33,6 +45,7 @@ class Config:
     version = 'C11'
     versand_separat = False
     cognos7: Cognos7Config
+    cognos11: Cognos11Config
     _cfg = {}
 
     def __init__(self, ini='GAPS.ini'):
@@ -110,4 +123,13 @@ class Config:
             self.cognos7.publish_dir = joinpath(self.portal_dir, 'daten'),
 
     def cognos11_config(self):
-        pass
+        self.cognos11 = Cognos11Config(
+            logs_dir=joinpath(self.log_dir, 'c11'),
+            reportoutput_dir=joinpath(self.portal_dir, 'ReportOutput'),
+            specs_dir=joinpath(self.system_dir, 'Report'),
+            webservice='http://localhost:9300/bi/',
+            config_dir=joinpath(self.xml_dir, 'c11'),
+            folders_file=joinpath(self.xml_dir, 'c11', 'folders.json'),
+            reports_file=joinpath(self.xml_dir, 'c11', 'reports.json'),
+            jobs_file=joinpath(self.xml_dir, 'c11', 'jobs.json')
+        )

pdf/pdf_merge.py

@@ -1,3 +1,4 @@
+import config
 from PyPDF2 import PdfMerger
 from pathlib import Path
 import re
@@ -6,7 +7,7 @@ import json
 
 def pdf_merge(base_dir: str, report_name):
     report_prefix = re.sub(r'\[([^\]]+)\]', '', report_name)
-    target = f"{base_dir}\\{report_prefix}_0.pdf"
+    target = f"{base_dir}/{report_prefix}_0.pdf"
     if Path(target).exists():
         Path(target).unlink()
     pdfs = [f for f in Path(base_dir).glob(report_prefix + '*.pdf')]
@@ -17,14 +18,18 @@ def pdf_merge(base_dir: str, report_name):
             merger.write(target)
 
 
-def get_reports(config_dir, folder):
-    filename = config_dir + '\\reports.json'
-    reports = json.load(open(filename, 'r'))
+def get_reports(reports_file, folder):
+    with open(reports_file, 'r') as frh:
+        reports = json.load(frh)
     return [r['name'] for r in reports if r['path'].startswith(folder)]
 
 
-if __name__ == '__main__':
-    reports = get_reports('C:\\GlobalCube\\Tasks\\gctools\\logs\\config', 'Team Content/ReportOutput')
+def merge_reports_in_folder(cfg: config.Config, folder='Team Content/ReportOutput'):
+    reports = get_reports(cfg.cognos11.reports_file, folder)
     for r in reports:
         print(r)
-        pdf_merge('C:\\GlobalCube\\ReportOutput\\raw', r)
+        pdf_merge(cfg.portal_dir + '/ReportOutput', r)
+
+
+if __name__ == '__main__':
+    merge_reports_in_folder(config.Config())

pdf/pdf_test.py

@@ -1,3 +1,4 @@
+import os
 import pdfplumber
 import re
 import json
@@ -32,6 +33,8 @@ def current_date_test(base_dir: str):
 
 
 def missing_data(base_dir: str):
+    os.makedirs(base_dir + '/leer', exist_ok=True)
+
     errors = []
     for f in Path(base_dir).glob('*.pdf'):
         with pdfplumber.open(str(f)) as pdf:
@@ -39,17 +42,17 @@ def missing_data(base_dir: str):
             text = pdf.pages[0].extract_text()
 
         # print(text)
-        if pages < 2 and (re.search(r'Daten nicht verf', text) or text.endswith('unprod.abw.') or (len(text) < 650 and text.count('\n') < 8)):
+        if pages < 2 and any([
+            re.search(r'Daten nicht verf', text),
+            text.endswith('unprod.abw.'),
+            (len(text) < 650 and text.count('\n') < 8)
+        ]):
             errors.append(f.name)
-            f.rename(base_dir + '/../leer/' + f.name)
+            f.rename(base_dir + '/leer/' + f.name)
 
     return errors
 
 
-def main():
+if __name__ == '__main__':
     # current_date_test('C:\\GAPS_BMW\\Portal\\Publish\\daten\\GAPS_BMW_neu')
     missing_data('C:\\GlobalCube\\ReportOutput\\raw')
-
-
-if __name__ == '__main__':
-    main()