# from itertools import chain import json from pathlib import Path import pandas as pd from bs4 import BeautifulSoup from collections import defaultdict import re import sys sys.path.append(sys.path[0] + '\\..') from gctools.config import Config def get_path_info(base_dir): path_info_df = pd.read_csv(base_dir + '/logs/status/path_info.csv', sep=';', encoding='latin-1', converters={'process': str}) path_info_df.rename(columns={'name': 'filename'}, inplace=True) path_info_df['filename'] = path_info_df['filename'].str.lower() path_info_df.set_index('filename', inplace=True, drop=False) return path_info_df.to_dict(orient='index') def get_cubes(base_dir, cfg: Config, path_info, cubes_models): ver_files = [k for k in path_info.keys() if re.search(r'\\cubes\\.*\.ver', k)] cubes = {} for file in ver_files: match = re.search(r'\\cubes\\(.*)__\d+\.ver$', file) cube_name = match[1] cube_subdir = f"{file[:-4]}\\{cube_name}.mdc" cube_out = f"{cfg.system_dir}\\cube_out\\{cube_name}.mdc" cubes[cube_name] = { 'deployed_mdc': path_info[cube_subdir], 'cube_out_mdc': path_info[cube_out], 'model': cubes_models[cube_name], 'errors': [] } return cubes def get_models(base_dir, cfg: Config, path_info, fm_sources): models = {} for file in Path(base_dir + '\\config\\models').glob('*.log'): with open(file, 'r') as frh: model_infos = frh.read().lower().replace('"', '').replace(',', '').split('--') datasources = model_infos[1].split('\n') datasources = [d for d in datasources if d != ''] fm_src = [] fm_package = None if '[' in datasources[0]: fm_package = datasources.pop(0).upper() fm_src = datasources datasources = [] for src in fm_src: if src in fm_sources: datasources.extend(fm_sources[src]) datasources = sorted(set(datasources)) cube = re.search(r'\\cube_out\\(.*)\.mdc', model_infos[0])[1] models[file.name[:-8].lower()] = { 'framework_manager': 'J' if fm_package else 'N', 'fm_package': fm_package, 'fm_sources': fm_src, 'datasources': datasources, 'cube': cube, 'logfile_tasks': path_info.get(cfg.portal_dir + '\\tasks\\logs\\' + file.name[:-8] + '.log', {'mtime': '0'}), 'logfile_system': path_info.get(cfg.system_dir + '\\logs\\' + file.name[:-8] + '.log', {'mtime': '0'}), 'model_file': path_info.get(cfg.system_dir + '\\models\\' + file.name[:-4], {'mtime': '0'}), 'model_file_filled': path_info.get(cfg.system_dir + '\\models\\gefuellt\\' + file.name[:-4], {'mtime': '0'}), } return models def get_database_info(base_dir, cfg: Config): db_info_df = pd.read_csv(base_dir + '/logs/status/db_info.csv', sep=';', encoding='latin-1') db_info_df = db_info_df[db_info_df['DatabaseName'] == 'GC'] db_info_df['table'] = db_info_df['TableName'].str.lower() db_info_df.set_index('table', inplace=True) return db_info_df.to_dict(orient='index') def get_fm_sources(base_dir, cfg): bs = BeautifulSoup(open(base_dir + '\\config\\fm\\model.xml', 'r'), 'xml') sources = defaultdict(list) for item in bs.find_all('queryItem'): p = item.parent.parent.find('name').string if item.parent.parent.name == 'folder': p = item.parent.parent.parent.find('name').string parent = "[{0}].[{1}]".format( p, item.parent.find('name').string ) src = '' exp = '' if item.expression: if item.expression.refobj: src = item.expression.refobj.string else: exp = item.expression.string elif item.externalName: exp = item.externalName.string sources[parent].append((item.find('name').string, src, exp)) interface = {} for k, fields in sources.items(): if '[Schnittstelle]' not in k: continue key = k.split('.')[-1][1:-1].lower() links = [] for field in fields: links.append(follow_links(sources, field, '')) interface[key] = sorted(list(set([re.search(r'\.\[(.*)\]$', e)[1].lower() for e in links if '[Import]' in e]))) return interface def follow_links(sources, field, value): if field[1] == '': if field[2] == field[0]: return value return field[2] match = re.search(r'(\[.*\]\.\[.*\])\.\[(.*)\]', field[1]) key1 = match[1] val1 = match[2] if key1 in sources: for field2 in sources[key1]: if field2[0] != val1: continue return follow_links(sources, field2, key1) return key1 def get_datasources(base_dir, cfg, path_info): all_datasources = set([re.search(r'\\iqd\\.*\\(.*)\.imr', k)[1] for k in path_info.keys() if re.search(r'\\iqd\\.*\\.*\.imr', k)]) datasources = {} for ds in all_datasources: ds_search = f'\\{ds}.imr' imr_files = [k for k in path_info.keys() if re.search(r'\\iqd\\.*\.imr', k) and ds_search in k and 'austausch' not in k] if len(imr_files) == 0: imr_file = '0.imr' else: imr_file = imr_files.pop(0) datasources[ds] = { 'imr_file': path_info.get(imr_file, {'mtime': '0'}), 'iqd_file': path_info.get(imr_file[:-4] + '.iqd', {'mtime': '0'}), 'csv_file': path_info.get(cfg.system_dir + '\\export\\' + ds + '.csv', {'mtime': '0'}), 'csv_file_iqd_folder': path_info.get(imr_file[:-4] + '.csv', {'mtime': '0'}), 'duplicates': imr_files } return datasources def cubes_to_models(models): models_sort = sorted([(v.get('logfile_tasks')['mtime'], v.get('logfile_system')['mtime'], v.get('model_file_filled')['mtime'], v.get('model_file')['mtime'], k, v['cube']) for k, v in models.items()]) result = {} for m in models_sort: result[m[5]] = m[4] return result def main(): base_dir = 'status_server\\unzipped' cfg = Config(str(Path(base_dir + '\\gaps.ini').absolute())) # Dateiliste path_info = get_path_info(base_dir) # random_bat_file = [k for k in path_info.keys() if re.search(r'\\Tasks\\.*\.bat', k)][0] # portal_dir = re.search(r'(.*)\\Tasks\\.*\.bat', random_bat_file)[1] # print(path_info) # Liste aller Cubes result = {} # Modelle und Datenquellen result['fm_sources'] = get_fm_sources(base_dir, cfg) result['models'] = get_models(base_dir, cfg, path_info, result['fm_sources']) result['database'] = get_database_info(base_dir, cfg) result['datasources'] = get_datasources(base_dir, cfg, path_info) cubes_models = cubes_to_models(result['models']) result['cubes'] = get_cubes(base_dir, cfg, path_info, cubes_models) # Cubes aktuell? # Rest aktuell? json.dump(result, open('status_server/logs/export.json', 'w'), indent=2) if __name__ == '__main__': main()