status-server/app/tools/inspect_files.py

# from itertools import chain
import json
from pathlib import Path
import pandas as pd
from bs4 import BeautifulSoup
from collections import defaultdict
import re
import sys
from config import Config


def get_path_info(base_dir):
    path_info_df = pd.read_csv(base_dir + '/logs/status/path_info.csv', sep=';',
                               encoding='latin-1', converters={'process': str})
    path_info_df.rename(columns={'name': 'filename'}, inplace=True)
    path_info_df['filename'] = path_info_df['filename'].str.lower()
    path_info_df.set_index('filename', inplace=True, drop=False)
    return path_info_df.to_dict(orient='index')


def get_cubes(base_dir, cfg: Config, path_info, cubes_models):
    ver_files = [k for k in path_info.keys() if re.search(r'\\cubes\\.*\.ver', k)]
    cubes = {}
    for file in ver_files:
        match = re.search(r'\\cubes\\(.*)__\d+\.ver$', file)
        cube_name = match[1]
        cube_subdir = f"{file[:-4]}\\{cube_name}.mdc"
        cube_out = f"{cfg.system_dir}\\cube_out\\{cube_name}.mdc"

        cubes[cube_name] = {
            'deployed_mdc': path_info[cube_subdir],
            'cube_out_mdc': path_info[cube_out],
            'model': cubes_models[cube_name],
            'errors': []
        }
    return cubes


def get_models(base_dir, cfg: Config, path_info, fm_sources):
    models = {}
    for file in Path(base_dir + '\\config\\models').glob('*.log'):
        with open(file, 'r') as frh:
            model_infos = frh.read().lower().replace('"', '').replace(',', '').split('--')
        datasources = model_infos[1].split('\n')
        datasources = [d for d in datasources if d != '']
        fm_src = []
        fm_package = None
        if '[' in datasources[0]:
            fm_package = datasources.pop(0).upper()
            fm_src = datasources
            datasources = []
            for src in fm_src:
                if src in fm_sources:
                    datasources.extend(fm_sources[src])
            datasources = sorted(set(datasources))

        cube = re.search(r'\\cube_out\\(.*)\.mdc', model_infos[0])[1]
        models[file.name[:-8].lower()] = {
            'framework_manager': 'J' if fm_package else 'N',
            'fm_package': fm_package,
            'fm_sources': fm_src,
            'datasources': datasources,
            'cube': cube,
            'logfile_tasks': path_info.get(cfg.portal_dir + '\\tasks\\logs\\' + file.name[:-8] + '.log', {'mtime': '0'}),
            'logfile_system': path_info.get(cfg.system_dir + '\\logs\\' + file.name[:-8] + '.log', {'mtime': '0'}),
            'model_file': path_info.get(cfg.system_dir + '\\models\\' + file.name[:-4], {'mtime': '0'}),
            'model_file_filled': path_info.get(cfg.system_dir + '\\models\\gefuellt\\' + file.name[:-4], {'mtime': '0'}),
        }
    return models


def get_database_info(base_dir, cfg: Config):
    db_info_df = pd.read_csv(base_dir + '/logs/status/db_info.csv', sep=';',
                             encoding='latin-1')
    db_info_df = db_info_df[db_info_df['DatabaseName'] == 'GC']
    db_info_df['table'] = db_info_df['TableName'].str.lower()
    db_info_df.set_index('table', inplace=True)
    return db_info_df.to_dict(orient='index')


def get_fm_sources(base_dir, cfg):
    bs = BeautifulSoup(open(base_dir + '\\config\\fm\\model.xml', 'r'), 'xml')
    sources = defaultdict(list)
    for item in bs.find_all('queryItem'):
        p = item.parent.parent.find('name').string
        if item.parent.parent.name == 'folder':
            p = item.parent.parent.parent.find('name').string
        parent = "[{0}].[{1}]".format(
            p,
            item.parent.find('name').string
        )
        src = ''
        exp = ''
        if item.expression:
            if item.expression.refobj:
                src = item.expression.refobj.string
            else:
                exp = item.expression.string
        elif item.externalName:
            exp = item.externalName.string
        sources[parent].append((item.find('name').string, src, exp))
    interface = {}
    for k, fields in sources.items():
        if '[Schnittstelle]' not in k:
            continue
        key = k.split('.')[-1][1:-1].lower()
        links = []
        for field in fields:
            links.append(follow_links(sources, field, ''))
        interface[key] = sorted(list(set([re.search(r'\.\[(.*)\]$', e)[1].lower() for e in links if '[Import]' in e])))

    return interface


def follow_links(sources, field, value):
    if field[1] == '':
        if field[2] == field[0]:
            return value
        return field[2]
    match = re.search(r'(\[.*\]\.\[.*\])\.\[(.*)\]', field[1])
    key1 = match[1]
    val1 = match[2]
    if key1 in sources:
        for field2 in sources[key1]:
            if field2[0] != val1:
                continue
            return follow_links(sources, field2, key1)
    return key1


def get_datasources(base_dir, cfg, path_info):
    all_datasources = set([re.search(r'\\iqd\\.*\\(.*)\.imr', k)[1]
                           for k in path_info.keys()
                           if re.search(r'\\iqd\\.*\\.*\.imr', k)])
    datasources = {}
    for ds in all_datasources:
        ds_search = f'\\{ds}.imr'
        imr_files = [k for k in path_info.keys()
                     if re.search(r'\\iqd\\.*\.imr', k) and ds_search in k
                     and 'austausch' not in k]
        if len(imr_files) == 0:
            imr_file = '0.imr'
        else:
            imr_file = imr_files.pop(0)
        datasources[ds] = {
            'imr_file': path_info.get(imr_file, {'mtime': '0'}),
            'iqd_file': path_info.get(imr_file[:-4] + '.iqd', {'mtime': '0'}),
            'csv_file': path_info.get(cfg.system_dir + '\\export\\' + ds + '.csv', {'mtime': '0'}),
            'csv_file_iqd_folder': path_info.get(imr_file[:-4] + '.csv', {'mtime': '0'}),
            'duplicates': imr_files
        }
    return datasources


def cubes_to_models(models):
    models_sort = sorted([(v.get('logfile_tasks')['mtime'],
                           v.get('logfile_system')['mtime'],
                           v.get('model_file_filled')['mtime'],
                           v.get('model_file')['mtime'],
                           k, v['cube']) for k, v in models.items()])
    result = {}
    for m in models_sort:
        result[m[5]] = m[4]
    return result


def main():
    base_dir = 'app/temp/unzipped/loeffler-c11_2022-07-08_112547'
    cfg = Config(str(Path(base_dir + '\\gaps.ini').absolute()))
    # Dateiliste
    path_info = get_path_info(base_dir)
    # random_bat_file = [k for k in path_info.keys() if re.search(r'\\Tasks\\.*\.bat', k)][0]
    # portal_dir = re.search(r'(.*)\\Tasks\\.*\.bat', random_bat_file)[1]
    # print(path_info)
    # Liste aller Cubes
    result = {}

    # Modelle und Datenquellen
    result['fm_sources'] = get_fm_sources(base_dir, cfg)
    result['models'] = get_models(base_dir, cfg, path_info, result['fm_sources'])
    result['database'] = get_database_info(base_dir, cfg)
    result['datasources'] = get_datasources(base_dir, cfg, path_info)
    cubes_models = cubes_to_models(result['models'])
    result['cubes'] = get_cubes(base_dir, cfg, path_info, cubes_models)
    # Cubes aktuell?
    # Rest aktuell?
    json.dump(result, open('app/temp/logs/export.json', 'w'), indent=2)


if __name__ == '__main__':
    main()