# from itertools import chain import json from pathlib import Path import pandas as pd from bs4 import BeautifulSoup from collections import defaultdict import re import sys from config import Config def get_path_info(base_dir): path_info_df = pd.read_csv(base_dir + '/logs/status/path_info.csv', sep=';', encoding='latin-1', converters={'process': str}) path_info_df.rename(columns={'name': 'filename'}, inplace=True) path_info_df['filename'] = path_info_df['filename'].str.lower() path_info_df.set_index('filename', inplace=True, drop=False) return path_info_df.to_dict(orient='index') def get_cubes(base_dir, cfg: Config, path_info, cubes_models): ver_files = [k for k in path_info.keys() if re.search(r'\\cubes\\.*\.ver', k)] cubes = {} for file in ver_files: match = re.search(r'\\cubes\\(.*)__\d+\.ver$', file) cube_name = match[1] cube_subdir = f"{file[:-4]}\\{cube_name}.mdc" cube_out = f"{cfg.system_dir}\\cube_out\\{cube_name}.mdc" cubes[cube_name] = { 'deployed_mdc': path_info[cube_subdir], 'cube_out_mdc': path_info[cube_out], 'model': cubes_models[cube_name], 'errors': [] } return cubes def get_models(base_dir, cfg: Config, path_info, fm_sources): models = {} for file in Path(base_dir + '\\config\\models').glob('*.log'): with open(file, 'r') as frh: model_infos = frh.read().lower().replace('"', '').replace(',', '').split('--') datasources = model_infos[1].split('\n') datasources = [d for d in datasources if d != ''] fm_src = [] fm_package = None if '[' in datasources[0]: fm_package = datasources.pop(0).upper() fm_src = datasources datasources = [] for src in fm_src: if src in fm_sources: datasources.extend(fm_sources[src]) datasources = sorted(set(datasources)) cube = re.search(r'\\cube_out\\(.*)\.mdc', model_infos[0])[1] models[file.name[:-8].lower()] = { 'framework_manager': 'J' if fm_package else 'N', 'fm_package': fm_package, 'fm_sources': fm_src, 'datasources': datasources, 'cube': cube, 'logfile_tasks': path_info.get(cfg.portal_dir + '\\tasks\\logs\\' + file.name[:-8] + '.log', {'mtime': '0'}), 'logfile_system': path_info.get(cfg.system_dir + '\\logs\\' + file.name[:-8] + '.log', {'mtime': '0'}), 'model_file': path_info.get(cfg.system_dir + '\\models\\' + file.name[:-4], {'mtime': '0'}), 'model_file_filled': path_info.get(cfg.system_dir + '\\models\\gefuellt\\' + file.name[:-4], {'mtime': '0'}), } return models def get_database_info(base_dir, cfg: Config): db_info_df = pd.read_csv(base_dir + '/logs/status/db_info.csv', sep=';', encoding='latin-1') db_info_df = db_info_df[db_info_df['DatabaseName'] == 'GC'] db_info_df['table'] = db_info_df['TableName'].str.lower() db_info_df.set_index('table', inplace=True) return db_info_df.to_dict(orient='index') def get_fm_sources(base_dir, cfg): bs = BeautifulSoup(open(base_dir + '\\config\\fm\\model.xml', 'r'), 'xml') sources = defaultdict(list) for item in bs.find_all('queryItem'): p = item.parent.parent.find('name').string if item.parent.parent.name == 'folder': p = item.parent.parent.parent.find('name').string parent = "[{0}].[{1}]".format( p, item.parent.find('name').string ) src = '' exp = '' if item.expression: if item.expression.refobj: src = item.expression.refobj.string else: exp = item.expression.string elif item.externalName: exp = item.externalName.string sources[parent].append((item.find('name').string, src, exp)) interface = {} for k, fields in sources.items(): if '[Schnittstelle]' not in k: continue key = k.split('.')[-1][1:-1].lower() links = [] for field in fields: links.append(follow_links(sources, field, '')) interface[key] = sorted(list(set([re.search(r'\.\[(.*)\]$', e)[1].lower() for e in links if '[Import]' in e]))) return interface def follow_links(sources, field, value): if field[1] == '': if field[2] == field[0]: return value return field[2] match = re.search(r'(\[.*\]\.\[.*\])\.\[(.*)\]', field[1]) key1 = match[1] val1 = match[2] if key1 in sources: for field2 in sources[key1]: if field2[0] != val1: continue return follow_links(sources, field2, key1) return key1 def get_datasources(base_dir, cfg, path_info): all_datasources = set([re.search(r'\\iqd\\.*\\(.*)\.imr', k)[1] for k in path_info.keys() if re.search(r'\\iqd\\.*\\.*\.imr', k)]) datasources = {} for ds in all_datasources: ds_search = f'\\{ds}.imr' imr_files = [k for k in path_info.keys() if re.search(r'\\iqd\\.*\.imr', k) and ds_search in k and 'austausch' not in k] if len(imr_files) == 0: imr_file = '0.imr' else: imr_file = imr_files.pop(0) datasources[ds] = { 'imr_file': path_info.get(imr_file, {'mtime': '0'}), 'iqd_file': path_info.get(imr_file[:-4] + '.iqd', {'mtime': '0'}), 'csv_file': path_info.get(cfg.system_dir + '\\export\\' + ds + '.csv', {'mtime': '0'}), 'csv_file_iqd_folder': path_info.get(imr_file[:-4] + '.csv', {'mtime': '0'}), 'duplicates': imr_files } return datasources def cubes_to_models(models): models_sort = sorted([(v.get('logfile_tasks')['mtime'], v.get('logfile_system')['mtime'], v.get('model_file_filled')['mtime'], v.get('model_file')['mtime'], k, v['cube']) for k, v in models.items()]) result = {} for m in models_sort: result[m[5]] = m[4] return result def main(): base_dir = 'status_server\\unzipped' cfg = Config(str(Path(base_dir + '\\gaps.ini').absolute())) # Dateiliste path_info = get_path_info(base_dir) # random_bat_file = [k for k in path_info.keys() if re.search(r'\\Tasks\\.*\.bat', k)][0] # portal_dir = re.search(r'(.*)\\Tasks\\.*\.bat', random_bat_file)[1] # print(path_info) # Liste aller Cubes result = {} # Modelle und Datenquellen result['fm_sources'] = get_fm_sources(base_dir, cfg) result['models'] = get_models(base_dir, cfg, path_info, result['fm_sources']) result['database'] = get_database_info(base_dir, cfg) result['datasources'] = get_datasources(base_dir, cfg, path_info) cubes_models = cubes_to_models(result['models']) result['cubes'] = get_cubes(base_dir, cfg, path_info, cubes_models) # Cubes aktuell? # Rest aktuell? json.dump(result, open('status_server/logs/export.json', 'w'), indent=2) if __name__ == '__main__': main()