IQD endlich in SQL verwertbar

tools/cognos7/iqd_convert.py

@@ -0,0 +1,95 @@
+from pathlib import Path
+import re
+import plac
+
+
+class IqdConverter:
+    commands = ['convert']
+    target_schema_by_type = {
+        'csv': 'staging',
+        'ims': 'ims'
+    }
+    output_dir = 'C:\\GlobalCube_LOCOSOFT\\System\\LOCOSOFT\\SQL\\schema\\LOCOSOFT\\views'
+
+    def convert(self, iqd_file, target_type='csv'):
+        query = self.get_query_from_iqdfile(iqd_file)
+        query = self.cleanup_query(query)
+
+        output_file = iqd_file[:-4] + '.sql'
+        with open(output_file, 'w') as fwh:
+            fwh.write(query)
+
+        table = Path(iqd_file).stem
+        schema = self.target_schema_by_type[target_type]
+
+        create_view = "SET QUOTED_IDENTIFIER ON \nGO\nSET ANSI_NULLS ON \nGO\n" + \
+            f"CREATE VIEW {schema}.{table} AS\n\n{query}\n" + \
+            "GO\nSET QUOTED_IDENTIFIER OFF \nGO\nSET ANSI_NULLS OFF \nGO\n\nGO"
+
+        view_file = f"{self.output_dir}\\{schema}.{table}.sql"
+        with open(view_file, 'w') as fwh:
+            fwh.write(create_view)
+
+    def get_query_from_iqdfile(self, iqd_file):
+        if iqd_file[-4:].lower() == '.imr':
+            iqd_file = iqd_file[:-4] + '.iqd'
+        iqd = open(iqd_file, 'r', encoding='latin-1').read()
+        res = re.findall(r'BEGIN SQL\n(.*)\n\nEND SQL', iqd, re.MULTILINE | re.DOTALL)
+        query = ''
+        if len(res) > 0:
+            query = res[0]
+
+        columns = re.findall(r'COLUMN,\d+,(.*)', iqd)
+        col_keys = [f'c{i}' for i in range(len(columns), 0, -1)]
+        col_names = [f'"{c}"' for c in reversed(columns)]
+
+        for col_key, col_name in zip(col_keys, col_names):
+            query = re.sub(col_key + r'([^\d])', col_name + r'\1', query)
+        return query
+
+    def cleanup_query(self, query):
+        query = re.sub(r' from (\d+) for (\d+)\)', r', \1, \2)', query)
+        query = query.replace(' || ', ' + ')
+        query = query.replace('truncate(', 'rtrim(')
+        query = query.replace('char_length(', 'len(')
+        query = query.replace('ascii(', 'convert(varchar(50), ')
+        query = query.replace('extract(DAY FROM ', 'day(')
+        query = query.replace('extract(MONTH FROM ', 'month(')
+        query = query.replace('extract(YEAR FROM ', 'year(')
+        query = query.replace('cdatetime(', 'convert(datetime, ')
+        query = query.replace('cast_float(', 'convert(float, ')
+        query = query.replace('sy_right(', 'right(')
+        query = query.replace('od_left(', 'left(')
+        query = query.replace('length(', 'len(')
+        query = query.replace('{hour}', 'hour')
+        query = query.replace('{minute}', 'minute')
+        query = query.replace('{weekday}', 'weekday')
+        query = query.replace('@CURRENT_DATE', 'getdate()')
+        query = query.replace('cdate(', '(')
+
+        query = re.sub(r'intdiv\(([^\)]+)\,1\)', r'\1', query)
+        query = re.sub(r'XCOUNT\(([^\)]+) for ', r'COUNT(\1) OVER (partition by ', query)
+        query = re.sub(r'XSUM\(([^\)]+) for ', r'SUM(\1) OVER (partition by ', query)
+        query = re.sub(r'XMIN\(([^\)]+) for ', r'MIN(\1) OVER (partition by ', query)
+        query = re.sub(r'XMAX\(([^\)]+) for ', r'MAX(\1) OVER (partition by ', query)
+
+        query = re.sub(r'QSS\.\"[^\"]+\\([^\\]+)\.ims\"', r'"ims"."\1"', query)
+        query = re.sub(r"DATE '([\d-]+)'", r"convert(date, '\1')", query)
+        query = re.sub(r"TIMESTAMP '([\d\s\-\:\.]+)'", r"convert(datetime, '\1')", query)
+
+        query = re.sub(r'asciiz\(([^\,]*)\,\d+\)', r'convert(varchar(50), \1)', query)
+        query = re.sub(r'asciiz\(([^\+]*)\)', r'convert(varchar(50), \1)', query)
+        query = re.sub(r'convert\(varchar\(50\)\, ([^,]+)\,\d+\)', r'convert(varchar(50), \1)', query)
+        query = query.replace('cdate((convert(float, ', 'convert(datetime, ((')
+
+        query = re.sub(r'(order by .*)', r'-- \1', query)
+        return query
+
+    def run_folder(base_dir: str):
+        files = sorted([(f.stat().st_mtime, f) for f in Path(base_dir).rglob('*.iqd')])
+        print(files)
+
+
+if __name__ == '__main__':
+    IqdConverter().convert('C:\\GlobalCube_LOCOSOFT\\System\\LOCOSOFT\\IQD\\NW\\LOC_Belege_NW_GW_VK_Stk_FIBU.iqd')
+    # plac.Interpreter.call(IqdConverter)

tools/cognos7/schema_ini_convert.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+
+
+def schema_convert(base_dir):
+    base_dir = Path(base_dir).absolute()
+    schema_file = base_dir / 'schema.ini'
+    with open(schema_file, 'r', encoding='latin-1') as frh:
+        schema = frh.read()
+    # schema = schema.replace('\n\n', '\n').replace('\n\n', '\n')
+    tables = dict([convert_table_info(t) for t in schema[1:].split('\n[')])
+
+    for table, details in tables.items():
+        table_file = base_dir / table
+        if 'schema.ini' in table or not table_file.exists():
+            continue
+        format_file = base_dir / (table + '.xml')
+
+        record = []
+        row = []
+        for i, c in details['columns'].items():
+            col_name = c.split(' ')[0]
+            record.append(f'  <FIELD ID="{i}" xsi:type="CharTerm" TERMINATOR=";" MAX_LENGTH="255" COLLATION="SQL_Latin1_General_CP1_CI_AS"/>')
+            row.append(f'  <COLUMN SOURCE="{i}" NAME="{col_name}" xsi:type="SQLVARYCHAR"/>')
+        record[-1] = record[-1].replace(';', '\\r\\n')
+
+        with open(format_file, 'w') as fwh:
+            fwh.write('<?xml version="1.0"?>\n<BCPFORMAT xmlns="http://schemas.microsoft.com/sqlserver/2004/bulkload/format" ')
+            fwh.write('xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">\n<RECORD>\n')
+            fwh.write('\n'.join(record))
+            fwh.write('\n</RECORD>\n<ROW>\n')
+            fwh.write('\n'.join(row))
+            fwh.write('\n</ROW>\n</BCPFORMAT>')
+    # print(tables)
+
+
+def convert_table_info(table_info):
+    info = table_info.split(']\n')
+    if len(info) < 2:
+        return ('', '')
+    details = {}
+    details['columns'] = {}
+    for key, value in ([row.split('=') for row in info[1].split('\n') if '=' in row]):
+        if key.lower() != 'colnameheader' and key.lower()[:3] == 'col':
+            details['columns'][key[3:]] = value
+    return (info[0], details)
+
+
+if __name__ == '__main__':
+    schema_convert('C:\\GlobalCube_LOCOSOFT\\GCStruct_SKR51\\Kontenrahmen')
+    # plac.Interpreter.call(IqdConverter)