Python-Projekte/status_server/inspect_files.py

# from itertools import chain
import json
from pathlib import Path
import pandas as pd
from bs4 import BeautifulSoup
from collections import defaultdict
import re
import sys
sys.path.append(sys.path[0] + '\\..')
from tools.config.config import Config


def get_path_info(base_dir):
    path_info_df = pd.read_csv(base_dir + '/logs/status/path_info.csv', sep=';',
                               encoding='latin-1', converters={'process': str})
    path_info_df.rename(columns={'name': 'filename'}, inplace=True)
    path_info_df['filename'] = path_info_df['filename'].str.lower()
    path_info_df.set_index('filename', inplace=True, drop=False)
    return path_info_df.to_dict(orient='index')


def get_cubes(base_dir, cfg: Config, path_info, cubes_models):
    ver_files = [k for k in path_info.keys() if re.search(r'\\cubes\\.*\.ver', k)]
    cubes = {}
    for file in ver_files:
        match = re.search(r'\\cubes\\(.*)__\d+\.ver$', file)
        cube_name = match[1]
        cube_subdir = f"{file[:-4]}\\{cube_name}.mdc"
        cube_out = f"{cfg.system_dir}\\cube_out\\{cube_name}.mdc"

        cubes[cube_name] = {
            'deployed_mdc': path_info[cube_subdir],
            'cube_out_mdc': path_info[cube_out],
            'model': cubes_models[cube_name],
            'errors': []
        }
    return cubes


def get_models(base_dir, cfg: Config, path_info, fm_sources):
    models = {}
    for file in Path(base_dir + '\\config\\models').glob('*.log'):
        with open(file, 'r') as frh:
            model_infos = frh.read().lower().replace('"', '').replace(',', '').split('--')
        datasources = model_infos[1].split('\n')
        datasources = [d for d in datasources if d != '']
        fm_src = []
        fm_package = None
        if '[' in datasources[0]:
            fm_package = datasources.pop(0).upper()
            fm_src = datasources
            datasources = []
            for src in fm_src:
                if src in fm_sources:
                    datasources.extend(fm_sources[src])
            datasources = sorted(set(datasources))

        cube = re.search(r'\\cube_out\\(.*)\.mdc', model_infos[0])[1]
        models[file.name[:-8].lower()] = {
            'framework_manager': 'J' if fm_package else 'N',
            'fm_package': fm_package,
            'fm_sources': fm_src,
            'datasources': datasources,
            'cube': cube,
            'logfile_tasks': path_info.get(cfg.portal_dir + '\\tasks\\logs\\' + file.name[:-8] + '.log', {'mtime': '0'}),
            'logfile_system': path_info.get(cfg.system_dir + '\\logs\\' + file.name[:-8] + '.log', {'mtime': '0'}),
            'model_file': path_info.get(cfg.system_dir + '\\models\\' + file.name[:-4], {'mtime': '0'}),
            'model_file_filled': path_info.get(cfg.system_dir + '\\models\\gefuellt\\' + file.name[:-4], {'mtime': '0'}),
        }
    return models


def get_database_info(base_dir, cfg: Config):
    db_info_df = pd.read_csv(base_dir + '/logs/status/db_info.csv', sep=';',
                             encoding='latin-1')
    db_info_df = db_info_df[db_info_df['DatabaseName'] == 'GC']
    db_info_df['table'] = db_info_df['TableName'].str.lower()
    db_info_df.set_index('table', inplace=True)
    return db_info_df.to_dict(orient='index')


def get_fm_sources(base_dir, cfg):
    bs = BeautifulSoup(open(base_dir + '\\config\\fm\\model.xml', 'r'), 'xml')
    sources = defaultdict(list)
    for item in bs.find_all('queryItem'):
        p = item.parent.parent.find('name').string
        if item.parent.parent.name == 'folder':
            p = item.parent.parent.parent.find('name').string
        parent = "[{0}].[{1}]".format(
            p,
            item.parent.find('name').string
        )
        src = ''
        exp = ''
        if item.expression:
            if item.expression.refobj:
                src = item.expression.refobj.string
            else:
                exp = item.expression.string
        elif item.externalName:
            exp = item.externalName.string
        sources[parent].append((item.find('name').string, src, exp))
    interface = {}
    for k, fields in sources.items():
        if '[Schnittstelle]' not in k:
            continue
        key = k.split('.')[-1][1:-1].lower()
        links = []
        for field in fields:
            links.append(follow_links(sources, field, ''))
        interface[key] = sorted(list(set([re.search(r'\.\[(.*)\]$', e)[1].lower() for e in links if '[Import]' in e])))

    return interface


def follow_links(sources, field, value):
    if field[1] == '':
        if field[2] == field[0]:
            return value
        return field[2]
    match = re.search(r'(\[.*\]\.\[.*\])\.\[(.*)\]', field[1])
    key1 = match[1]
    val1 = match[2]
    if key1 in sources:
        for field2 in sources[key1]:
            if field2[0] != val1:
                continue
            return follow_links(sources, field2, key1)
    return key1


def get_datasources(base_dir, cfg, path_info):
    all_datasources = set([re.search(r'\\iqd\\.*\\(.*)\.imr', k)[1]
                           for k in path_info.keys()
                           if re.search(r'\\iqd\\.*\\.*\.imr', k)])
    datasources = {}
    for ds in all_datasources:
        ds_search = f'\\{ds}.imr'
        imr_files = [k for k in path_info.keys()
                     if re.search(r'\\iqd\\.*\.imr', k) and ds_search in k
                     and 'austausch' not in k]
        if len(imr_files) == 0:
            imr_file = '0.imr'
        else:
            imr_file = imr_files.pop(0)
        datasources[ds] = {
            'imr_file': path_info.get(imr_file, {'mtime': '0'}),
            'iqd_file': path_info.get(imr_file[:-4] + '.iqd', {'mtime': '0'}),
            'csv_file': path_info.get(cfg.system_dir + '\\export\\' + ds + '.csv', {'mtime': '0'}),
            'csv_file_iqd_folder': path_info.get(imr_file[:-4] + '.csv', {'mtime': '0'}),
            'duplicates': imr_files
        }
    return datasources


def cubes_to_models(models):
    models_sort = sorted([(v.get('logfile_tasks')['mtime'],
                           v.get('logfile_system')['mtime'],
                           v.get('model_file_filled')['mtime'],
                           v.get('model_file')['mtime'],
                           k, v['cube']) for k, v in models.items()])
    result = {}
    for m in models_sort:
        result[m[5]] = m[4]
    return result


def main():
    base_dir = 'status_server\\unzipped'
    cfg = Config(str(Path(base_dir + '\\gaps.ini').absolute()))
    # Dateiliste
    path_info = get_path_info(base_dir)
    # random_bat_file = [k for k in path_info.keys() if re.search(r'\\Tasks\\.*\.bat', k)][0]
    # portal_dir = re.search(r'(.*)\\Tasks\\.*\.bat', random_bat_file)[1]
    # print(path_info)
    # Liste aller Cubes
    result = {}

    # Modelle und Datenquellen
    result['fm_sources'] = get_fm_sources(base_dir, cfg)
    result['models'] = get_models(base_dir, cfg, path_info, result['fm_sources'])
    result['database'] = get_database_info(base_dir, cfg)
    result['datasources'] = get_datasources(base_dir, cfg, path_info)
    cubes_models = cubes_to_models(result['models'])
    result['cubes'] = get_cubes(base_dir, cfg, path_info, cubes_models)
    # Cubes aktuell?
    # Rest aktuell?
    json.dump(result, open('status_server/logs/export.json', 'w'), indent=2)


if __name__ == '__main__':
    main()