C11 pdf export mit Fehlerunterdrückung

tools/cognos11/c11_api.py

@@ -1,6 +1,5 @@
 import base64
 import os
-import time
 import requests
 from requests_toolbelt.multipart import decoder
 import jinja2
@@ -8,6 +7,7 @@ import json
 import re
 from bs4 import BeautifulSoup
 from xml_prettify import prettify_xml
+import logging
 
 
 class c11_api:
@@ -19,7 +19,9 @@ class c11_api:
     headers = {}
     caf = ""
     cam = ""
-    reports = None
+    reports = []
+    folders = []
+    jobs = []
 
     def __init__(self, webservice="http://localhost:9300/bi/"):
         self.webservice = webservice
@@ -63,30 +65,73 @@ class c11_api:
         self.cam = self.generate_token(r.cookies["usersessionid"])
         return r.status_code
 
-    def report_list(self):
-        # "v1/objects/_dot_public_folders/items"
-        fields = "defaultName|id|ancestors"
-        filter = "type|analysis|interactiveReport|powerPlayReport|powerPlay8Report|powerPlay8ReportView|query|report|reportTemplate"
+    def get_folders(self):
+        if len(self.folders) == 0:
+            self.load_folder_list()
+        return self.folders
+
+    def load_folder_list(self, folder_id='_dot_public_folders', prefix='Team Content'):
+        res = self.session.get(f"{self.webservice}v1/objects/{folder_id}/items", headers=self.headers)
+        folder_list = res.json()['data']
+        for f in folder_list:
+            if f['type'] == 'folder':
+                folder = {
+                    'id': f['id'],
+                    'name': prefix + '/' + f['defaultName'].replace('/', '_')
+                }
+                self.folders.append(folder)
+                self.load_folder_list(folder['id'], folder['name'])
+            elif f['type'] == 'report':
+                report = {
+                    'id': f['id'],
+                    'name': f['defaultName'],
+                    'path': prefix
+                }
+                self.reports.append(report)
+            elif f['type'] == 'jobDefinition':
+                job = {
+                    'id': f['id'],
+                    'name': f['defaultName'],
+                    'path': prefix
+                }
+                job['details'] = self.get_job_details(job['id'])
+                self.jobs.append(job)
+
+    def get_job_details(self, job_id):
+        fields = ",".join([
+            'userInterfaces,disabled',
+            'runInAdvancedViewer,modificationTime,canBurst',
+            'defaultPortalAction',
+            'base.defaultName,tags,target.searchPath,target.disabled',
+            'options,base.options'
+        ])
+        res = self.session.get(f"{self.webservice}v1/objects/{job_id}?fields={fields}", headers=self.headers)
+        job = res.json()['data'][0]
+        job.pop('_meta', None)
+        job.pop('id', None)
+        job.pop('type', None)
+        job.pop('defaultName', None)
+
+        fields2 = ",".join([
+            r'id,displaySequence,stepObject{defaultName}',
+            r'stepObject{id},stepObject{parameters}',
+            r'stepObject{canBurst},options,parameters'
+        ])
         res = self.session.get(
-            f"{self.webservice}v1/search/cm?fields={fields}&results=1000&query=.&hide_internal=all&filter={filter}",
+            f"{self.webservice}v1/objects/{job_id}/items?types=jobStepDefinition&fields={fields2}",
             headers=self.headers
         )
-        # filename = self.log_dir + '/reports_error.log'
-        # os.makedirs(os.path.dirname(filename), exist_ok=True)
-        # with open(filename, "wb") as f:
-        #     f.write(res.content)
-        self.reports = res.json()['results']
-        for r in self.reports:
-            r['path'] = "/".join([a['defaultName'].replace('/', '_') for a in r['ancestors']])
-            r['name'] = r['defaultName']
-            del(r['ancestors'])
-            del(r['defaultName'])
-
-        return self.reports
+        steps = res.json()['data']
+        for s in steps:
+            s.pop('_meta', None)
+            if s['stepObject'] is not None:
+                s['report_id'] = s['stepObject'][0]['id']
+            s.pop('stepObject', None)
+        job['steps'] = steps
+        return job
 
     def get_report(self, report_id):
-        if self.reports is None:
-            self.reports = self.report_list()
+        self.get_folders()
         report = [r for r in self.reports if r['id'] == report_id]
         if len(report) == 0:
             return None
@@ -96,8 +141,7 @@ class c11_api:
         return report
 
     def get_reports_in_folder(self, folder, recursive=False):
-        if self.reports is None:
-            self.reports = self.report_list()
+        self.get_folders()
         if recursive:
             return [r for r in self.reports if r['path'].startswith(folder)]
         return [r for r in self.reports if r['path'] == folder]
@@ -118,18 +162,14 @@ class c11_api:
         if r.status_code == 500:
             bs = BeautifulSoup(r.text, 'xml')
             report['error'] = bs.find_all('messageString')[0].string
-            # time.sleep(2)
+            logging.error(report['error'])
             return report
 
         parts = decoder.MultipartDecoder.from_response(r).parts
 
-        # for i, p in enumerate(parts):
-        #     with open(f"export/{report['report']}_{i}.xml", "w") as f:
-        #         f.write(p.text.replace('\x81', ''))
-
         meta = {'required': {}, 'optional': {}}
         bs = BeautifulSoup(parts[1].content, 'lxml')
-        # print(bs.prettify())
+
         for sv in bs.find_all('selectvalue'):
             k = sv['parameter']
             req = 'required' if sv['required'] == 'true' else 'optional'
@@ -146,6 +186,12 @@ class c11_api:
         json.dump(meta, open(filename, 'w'), indent=2)
         report['meta'] = meta
         report['spec'] = parts[2].text
+        path = report['path'].replace('Team Content/ReportOutput', '')
+        report['filename'] = f"{self.export_dir}/{path}/{report['name']}.pdf"
+
+        report['params'] = list(re.findall(r'\[([^\]]+)\]', report['filename']))
+        for i, p in enumerate(report['params']):
+            report['filename'] = report['filename'].replace('[' + p + ']', '{' + str(i) + '}')
         return report
 
     def export_unstubbed(self, report_id):
@@ -201,36 +247,35 @@ class c11_api:
             'SOAPAction': 'http://www.ibm.com/xmlns/prod/cognos/reportService/202004/'
         }
 
-    def request_file(self, report_id, params, filename, format='PDF'):
+    def request_file(self, report_id, params, format='PDF'):
         report = self.get_report(report_id)
         headers = self.get_report_headers(report_id)
 
         soap = self.template.render({"caf": self.caf, "cam": self.cam,
                                      "report": report, "format": format,
                                      "prompt": 'false', "tracking": "", "params": params}).encode("utf-8")
-        try:
-            r = self.session.post(self.webservice + 'v1/reports', data=soap, headers=headers)
-        except UnicodeEncodeError:
-            filename = self.log_dir + '/' + os.path.basename(filename) + '.log'
-            os.makedirs(os.path.dirname(filename), exist_ok=True)
-            with open(filename, "w") as f:
-                f.write(soap)
-            return False
+        r = self.session.post(self.webservice + 'v1/reports', data=soap, headers=headers)
 
+        bs = BeautifulSoup(r.text, 'xml')
         if r.status_code == 200:
-            parts = decoder.MultipartDecoder.from_response(r).parts
-            filename = self.export_dir + filename
-            os.makedirs(os.path.dirname(filename), exist_ok=True)
-            with open(filename, "wb") as f:
-                f.write(parts[1].content)
-        else:
-            filename = self.log_dir + '/' + os.path.basename(filename) + '.log'
-            os.makedirs(os.path.dirname(filename), exist_ok=True)
-            with open(filename, "wb") as f:
-                f.write(r.content)
-        return True
+            try:
+                parts = decoder.MultipartDecoder.from_response(r).parts
+            except decoder.NonMultipartContentTypeException:
+                return 500, 'Timeout'
+            return 200, parts[1].content
+
+        error = bs.find_all('messageString')[0].string
+        logging.debug(error)
+        return r.status_code, error
 
 
 if __name__ == '__main__':
     api = c11_api()
     api.login()
+    folders = api.get_folders()
+    filename = 'C:/GlobalCube/Tasks/gctools/logs/config/folders.json'
+    json.dump(folders, open(filename, 'w'), indent=2)
+    filename = 'C:/GlobalCube/Tasks/gctools/logs/config/reports.json'
+    json.dump(api.reports, open(filename, 'w'), indent=2)
+    filename = 'C:/GlobalCube/Tasks/gctools/logs/config/jobs.json'
+    json.dump(api.jobs, open(filename, 'w'), indent=2)

tools/cognos11/pdf_export.py

@@ -1,70 +1,109 @@
 import json
+import logging
 import os
 from c11_api import c11_api
-import re
+from datetime import datetime
 
 
 class pdf_export:
     api: c11_api
+    log_dir = "C:/GlobalCube/Tasks/gctools/logs"
 
     def __init__(self, api):
         self.api = api
+        now = datetime.now().strftime('%Y%m%d_%H%M%S')
+        prot_file = f"{self.log_dir}/error_{now}.log"
+        logging.basicConfig(
+            filename=prot_file,
+            filemode='w',
+            encoding='utf-8',
+            level=logging.DEBUG,
+            force=True
+        )
 
-    def export_folder(self, folder, format='PDF'):
-        reports = api.get_reports_in_folder(folder, True)
+    def export_folder(self, folder='', format='PDF'):
+        if not folder.startswith('Team Content'):
+            folder = 'Team Content/ReportOutput/' + folder
+        reports = self.api.get_reports_in_folder(folder, True)
         for r in reports:
             print(r['name'])
             self.export_report(r['id'], format, folder=folder)
 
-    def export_report(self, report_id, format, params=None, folder=None):
-        report = api.get_report(report_id)
-
+    def export_report(self, report_id, format, folder=None):
         if format == 'PDF':
-            path = report['path'].replace(folder, '')
-            report['filename'] = f"{path}/{report['name']}.pdf"
-            params = list(re.findall(r'\[([^\]]+)\]', report['filename']))
-            for i, p in enumerate(params):
-                report['filename'] = report['filename'].replace('[' + p + ']', '{' + str(i) + '}')
-            return self.export_pdf(report, params)
+            return self.export_pdf(report_id, folder)
         if format == 'XML':
-            return api.export_unstubbed(report_id)
+            return self.api.export_unstubbed(report_id)
         return False
 
-    def export_pdf(self, report, params):
-        if len(params) == 0:
-            params = {}
+    def export_pdf(self, report_id, folder=None):
+        report = self.api.get_report(report_id)
+        if 'meta' not in report:
+            logging.warning(report['name'] + ' is not accessible!')
+            return False
+        params = {}
+        if len(report['meta']['required']) > 0:
+            if set(report['meta']['required'].keys()) != {'p_Von', 'p_Bis'}:
+                return False
+            params['p_Von'] = {'2022-10-12': '12.10.2022'}
+            params['p_Bis'] = {'2022-10-12': '12.10.2022'}
+            params['p_Zeitraum'] = {'Einzelne Monate': 'Einzelne Monate'}
+
+            for k, v in report['meta']['optional'].items():
+                if k in ['p_Zeit', 'p_Auswahl_Monate', 'p_12_Monate']:
+                    for k1, v1 in reversed(v.items()):
+                        if v1 != 'Invalid Dates':
+                            params[k] = {k1: v1}
+                            continue  # use last element only
+
+        if len(report['params']) == 0:
             filename = report['filename']
-            api.request_file(report['id'], params, filename)
+            self.request_and_save_file(report['id'], params, filename)
             return True
-        if len(params) == 1:
-            params = {}
+        if len(report['params']) == 1:
             filename = report['filename'].format('1')
-            api.request_file(report['id'], params, filename)
+            self.request_and_save_file(report['id'], params, filename)
 
-            key1 = params[0]
+            key1 = report['params'][0]
             for k1, v1 in report['meta']['optional'][key1].items():
                 filename = report['filename'].format(v1)
-                params = {key1: {k1: v1}}
-                api.request_file(report['id'], params, filename)
+                params[key1] = {k1: v1}
+                self.request_and_save_file(report['id'], params, filename)
             return True
-        if len(params) == 2:
-            key1, key2 = params
+        if len(report['params']) == 2:
+            key1, key2 = report['params']
             for k1, v1 in report['meta']['optional'][key1].items():
                 for k2, v2 in report['meta']['optional'][key2].items():
                     filename = report['filename'].format(v1, v2)
-                    params = {key1: {k1: v1}, key2: {k2: v2}}
-                    api.request_file(report['id'], params, filename)
+                    params[key1] = {k1: v1}
+                    params[key2] = {k2: v2}
+                    self.request_and_save_file(report['id'], params, filename)
             return True
 
+    def request_and_save_file(self, report_id, params, filename):
+        logging.debug(filename)
+        logging.debug(params)
+        status_code, content = self.api.request_file(report_id, params)
+        if status_code == 200:
+            os.makedirs(os.path.dirname(filename), exist_ok=True)
+            with open(filename, "wb") as f:
+                f.write(content)
+        else:
+            logging.warning(content)
+
+    def export_errors(self):
+        pdf.export_folder('Team Content', 'XML')
+        reports = self.api.get_reports_in_folder('Team Content', True)
+        errors = [r for r in reports if 'error' in r]
+        filename = 'C:/GlobalCube/Tasks/gctools/logs/config/report_errors.json'
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+        json.dump(errors, open(filename, 'w'), indent=2)
+
 
 if __name__ == '__main__':
     api = c11_api()
     api.login()
     pdf = pdf_export(api)
     # pdf.export_folder('Team Content/Verkauf/1. Gesamtverkauf', 'PDF')
-    pdf.export_folder('Team Content', 'XML')
-    reports = api.get_reports_in_folder('Team Content', True)
-    errors = [r for r in reports if 'error' in r]
-    filename = 'C:/GlobalCube/Tasks/gctools/logs/config/report_errors.json'
-    os.makedirs(os.path.dirname(filename), exist_ok=True)
-    json.dump(errors, open(filename, 'w'), indent=2)
+    pdf.export_folder('Team Content/Aftersales/1. Service')
+    # pdf.export_errors()

tools/cognos11/templates/get-report.xml

@@ -1,156 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<SOAP-ENV:Envelope
-	xmlns:SOAP-ENV='http://schemas.xmlsoap.org/soap/envelope/'
-	xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' SOAP-ENV:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/"
-	xmlns:SOAP-ENC='http://schemas.xmlsoap.org/soap/encoding/'
-	xmlns:xsd='http://www.w3.org/2001/XMLSchema'
-	xmlns:xs="http://www.w3.org/2001/XMLSchema"
-	xmlns:bus='http://developer.cognos.com/schemas/bibus/3/'
-	xmlns:rns1='http://developer.cognos.com/schemas/reportService/1'>
-	<SOAP-ENV:Header>
-		<bus:biBusHeader xsi:type="bus:biBusHeader">
-			<bus:CAM xsi:type="bus:CAM">
-				<authenticityToken xsi:type="xsd:base64Binary">{{cam}}</authenticityToken>
-			</bus:CAM>
-			<bus:CAF xsi:type="bus:CAF">
-				<contextID xsi:type="xsd:string">{{caf}}</contextID>
-			</bus:CAF>
-			<bus:hdrSession xsi:type="bus:hdrSession">
-				<bus:formFieldVars SOAP-ENC:arrayType="bus:formFieldVar[]" xsi:type="SOAP-ENC:Array">
-					<item xsi:type="bus:formFieldVar">
-						<name xsi:type="xsd:string">_ContextBlockSize</name>
-						<value xsi:type="xsd:string">1000000</value>
-					</item>
-					<item xsi:type="bus:formFieldVar">
-						<name xsi:type="xsd:string">ignoreXHTMLStrict</name>
-						<value xsi:type="xsd:string">true</value>
-					</item>
-				</bus:formFieldVars>
-			</bus:hdrSession>
-			<bus:userPreferenceVars SOAP-ENC:arrayType="bus:userPreferenceVar[]" xsi:type="SOAP-ENC:Array">
-				<item>
-					<bus:name xsi:type="xsd:string">productLocale</bus:name>
-					<bus:value xsi:type="xsd:string">de</bus:value>
-				</item>
-				<item>
-					<bus:name xsi:type="xsd:string">contentLocale</bus:name>
-					<bus:value xsi:type="xsd:string">de-de</bus:value>
-				</item>
-			</bus:userPreferenceVars>
-			<bus:dispatcherTransportVars xsi:type="SOAP-ENC:Array" SOAP-ENC:arrayType="bus:dispatcherTransportVar[]">
-				<item xsi:type="bus:dispatcherTransportVar">
-					<name xsi:type="xsd:string">rs</name>
-					<value xsi:type="xsd:string">true</value>
-				</item>
-			</bus:dispatcherTransportVars>
-			<bus:tracking
-				xmlns:bus="http://developer.cognos.com/schemas/bibus/3/"
-				xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="bus:tracking">{{tracking}}</bus:tracking>
-		</bus:biBusHeader>
-	</SOAP-ENV:Header>
-	<SOAP-ENV:Body>
-		<rns1:run>
-			<bus:objectPath xsi:type="bus:searchPathSingleObject">storeID(&quot;{{report.id}}&quot;)</bus:objectPath>
-			<bus:parameterValues
-				xmlns:bus='http://developer.cognos.com/schemas/bibus/3/'
-				xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
-				xmlns:SOAP-ENC='http://schemas.xmlsoap.org/soap/encoding/' SOAP-ENC:arrayType="bus:parameterValue[{{report.params|count}}]" xsi:type="SOAP-ENC:Array">
-{% for key, p in report.params.items() %}
-				<item xsi:type="bus:parameterValue">
-					<bus:name xsi:type="xs:string">{{key}}</bus:name>
-					<bus:value xsi:type="SOAP-ENC:Array" SOAP-ENC:arrayType="bus:parmValueItem[{{p|count}}]">
-{% for item in p %}
-						<item xsi:type="bus:simpleParmValueItem">
-							<bus:inclusive xsi:type="xs:boolean">true</bus:inclusive>
-							<bus:display xsi:type="xs:string">{{item.display}}</bus:display>
-							<bus:use xsi:type="xs:string">{{item.use}}</bus:use>
-						</item>
-{% endfor %}
-					</bus:value>
-				</item>
-{% endfor %}
-			</bus:parameterValues>
-			<bus:options SOAP-ENC:arrayType="bus:option[]" xsi:type="SOAP-ENC:Array">
-				<item xsi:type="bus:genericOptionAnyURI">
-					<bus:name xsi:type="xsd:string">runOptionEnum#globalParameters</bus:name>
-					<bus:value xsi:type="xsd:string">[]</bus:value>
-				</item>
-				<item xsi:type="bus:asynchOptionInt">
-					<bus:name xsi:type="bus:asynchOptionEnum">primaryWaitThreshold</bus:name>
-					<bus:value xsi:type="xsd:int">500</bus:value>
-				</item>
-				<item xsi:type="bus:asynchOptionInt">
-					<bus:name xsi:type="bus:asynchOptionEnum">secondaryWaitThreshold</bus:name>
-					<bus:value xsi:type="xsd:int">30</bus:value>
-				</item>
-				<item xsi:type="bus:runOptionStringArray">
-					<bus:name xsi:type="bus:runOptionEnum">outputFormat</bus:name>
-					<bus:value xsi:type="SOAP-ENC:Array" SOAP-ENC:arrayType="xsd:string[1]">
-						<item>{{format}}</item>
-					</bus:value>
-				</item>
-				<item xsi:type="bus:asynchOptionEncoding">
-					<bus:name xsi:type="bus:asynchOptionEnum">attachmentEncoding</bus:name>
-					<bus:value xsi:type="bus:encodingEnum">MIME</bus:value>
-				</item>
-				<item xsi:type="bus:runOptionAnyURI">
-					<bus:name xsi:type="bus:runOptionEnum">xslURL</bus:name>
-					<bus:value xsi:type="xsd:string">V5html_viewer.xsl</bus:value>
-				</item>
-				<item xsi:type="bus:runOptionString">
-					<bus:name xsi:type="bus:runOptionEnum">promptFormat</bus:name>
-					<bus:value xsi:type="xsd:string">XHTMLFRGMT</bus:value>
-				</item>
-				<item xsi:type="bus:runOptionBoolean">
-					<bus:name xsi:type="bus:runOptionEnum">prompt</bus:name>
-					<bus:value xsi:type="xsd:boolean">{{prompt}}</bus:value>
-				</item>
-				<item xsi:type="bus:runOptionAnyURI">
-					<bus:name xsi:type="bus:runOptionEnum">outputLocation</bus:name>
-					<bus:value xsi:type="xsd:string">http://developer.cognos.com/ceba/constants/temporaryObjectLocationEnum#serverFileSystem</bus:value>
-				</item>
-				<item xsi:type="bus:runOptionData">
-					<bus:name xsi:type="bus:runOptionEnum">data</bus:name>
-					<bus:value xsi:type="bus:dataEnum">runWithAllData</bus:value>
-				</item>
-				<item xsi:type="bus:genericOptionBoolean">
-					<bus:name xsi:type="xsd:string">http://developer.cognos.com/ceba/constants/systemOptionEnum#accessibilityFeatures</bus:name>
-					<bus:value xsi:type="xsd:boolean">false</bus:value>
-				</item>
-				<item xsi:type="bus:genericOptionBoolean">
-					<bus:name xsi:type="xsd:string">http://developer.cognos.com/ceba/constants/biDirectionalOptionEnum#biDirectionalFeaturesEnabled</bus:name>
-					<bus:value xsi:type="xsd:boolean">false</bus:value>
-				</item>
-				<item xsi:type="bus:runOptionBoolean">
-					<bus:name xsi:type="bus:runOptionEnum">returnOutputWhenAvailable</bus:name>
-					<bus:value xsi:type="xsd:boolean">true</bus:value>
-				</item>
-				<item xsi:type="bus:runOptionNameValueArray">
-					<bus:name xsi:type="bus:runOptionEnum">xslParameters</bus:name>
-					<bus:value xsi:type="SOAP-ENC:Array" SOAP-ENC:arrayType="bus:nameValue[]">
-						<item xsi:type="bus:nameValue">
-							<name xsi:type="xsd:string">CVGateway</name>
-							<value xsi:type="xsd:string">../v1/disp</value>
-						</item>
-						<item xsi:type="bus:nameValue">
-							<name xsi:type="xsd:string">renderIntermediateXML</name>
-							<value xsi:type="xsd:string">false</value>
-						</item>
-						<item xsi:type="bus:nameValue">
-							<name xsi:type="xsd:string">renderEnvironment</name>
-							<value xsi:type="xsd:string">false</value>
-						</item>
-					</bus:value>
-				</item>
-				<item xsi:type="bus:genericOptionBoolean">
-					<bus:name xsi:type="xsd:string">http://developer.cognos.com/ceba/constants/runOptionEnum#interactive</bus:name>
-					<bus:value xsi:type="xsd:boolean">true</bus:value>
-				</item>
-				<item xsi:type="bus:genericOptionAnyURI">
-					<bus:name xsi:type="xsd:string">http://developer.cognos.com/ceba/constants/runOptionEnum#promptXslUrl</bus:name>
-					<bus:value xsi:type="xsd:string">V5html_viewer.xsl</bus:value>
-				</item>
-			</bus:options>
-		</rns1:run>
-	</SOAP-ENV:Body>
-</SOAP-ENV:Envelope>