|
|
@@ -0,0 +1,191 @@
|
|
|
+# from itertools import chain
|
|
|
+import json
|
|
|
+from pathlib import Path
|
|
|
+import pandas as pd
|
|
|
+from bs4 import BeautifulSoup
|
|
|
+from collections import defaultdict
|
|
|
+import re
|
|
|
+import sys
|
|
|
+from config import Config
|
|
|
+
|
|
|
+
|
|
|
+def get_path_info(base_dir):
|
|
|
+ path_info_df = pd.read_csv(base_dir + '/logs/status/path_info.csv', sep=';',
|
|
|
+ encoding='latin-1', converters={'process': str})
|
|
|
+ path_info_df.rename(columns={'name': 'filename'}, inplace=True)
|
|
|
+ path_info_df['filename'] = path_info_df['filename'].str.lower()
|
|
|
+ path_info_df.set_index('filename', inplace=True, drop=False)
|
|
|
+ return path_info_df.to_dict(orient='index')
|
|
|
+
|
|
|
+
|
|
|
+def get_cubes(base_dir, cfg: Config, path_info, cubes_models):
|
|
|
+ ver_files = [k for k in path_info.keys() if re.search(r'\\cubes\\.*\.ver', k)]
|
|
|
+ cubes = {}
|
|
|
+ for file in ver_files:
|
|
|
+ match = re.search(r'\\cubes\\(.*)__\d+\.ver$', file)
|
|
|
+ cube_name = match[1]
|
|
|
+ cube_subdir = f"{file[:-4]}\\{cube_name}.mdc"
|
|
|
+ cube_out = f"{cfg.system_dir}\\cube_out\\{cube_name}.mdc"
|
|
|
+
|
|
|
+ cubes[cube_name] = {
|
|
|
+ 'deployed_mdc': path_info[cube_subdir],
|
|
|
+ 'cube_out_mdc': path_info[cube_out],
|
|
|
+ 'model': cubes_models[cube_name],
|
|
|
+ 'errors': []
|
|
|
+ }
|
|
|
+ return cubes
|
|
|
+
|
|
|
+
|
|
|
+def get_models(base_dir, cfg: Config, path_info, fm_sources):
|
|
|
+ models = {}
|
|
|
+ for file in Path(base_dir + '\\config\\models').glob('*.log'):
|
|
|
+ with open(file, 'r') as frh:
|
|
|
+ model_infos = frh.read().lower().replace('"', '').replace(',', '').split('--')
|
|
|
+ datasources = model_infos[1].split('\n')
|
|
|
+ datasources = [d for d in datasources if d != '']
|
|
|
+ fm_src = []
|
|
|
+ fm_package = None
|
|
|
+ if '[' in datasources[0]:
|
|
|
+ fm_package = datasources.pop(0).upper()
|
|
|
+ fm_src = datasources
|
|
|
+ datasources = []
|
|
|
+ for src in fm_src:
|
|
|
+ if src in fm_sources:
|
|
|
+ datasources.extend(fm_sources[src])
|
|
|
+ datasources = sorted(set(datasources))
|
|
|
+
|
|
|
+ cube = re.search(r'\\cube_out\\(.*)\.mdc', model_infos[0])[1]
|
|
|
+ models[file.name[:-8].lower()] = {
|
|
|
+ 'framework_manager': 'J' if fm_package else 'N',
|
|
|
+ 'fm_package': fm_package,
|
|
|
+ 'fm_sources': fm_src,
|
|
|
+ 'datasources': datasources,
|
|
|
+ 'cube': cube,
|
|
|
+ 'logfile_tasks': path_info.get(cfg.portal_dir + '\\tasks\\logs\\' + file.name[:-8] + '.log', {'mtime': '0'}),
|
|
|
+ 'logfile_system': path_info.get(cfg.system_dir + '\\logs\\' + file.name[:-8] + '.log', {'mtime': '0'}),
|
|
|
+ 'model_file': path_info.get(cfg.system_dir + '\\models\\' + file.name[:-4], {'mtime': '0'}),
|
|
|
+ 'model_file_filled': path_info.get(cfg.system_dir + '\\models\\gefuellt\\' + file.name[:-4], {'mtime': '0'}),
|
|
|
+ }
|
|
|
+ return models
|
|
|
+
|
|
|
+
|
|
|
+def get_database_info(base_dir, cfg: Config):
|
|
|
+ db_info_df = pd.read_csv(base_dir + '/logs/status/db_info.csv', sep=';',
|
|
|
+ encoding='latin-1')
|
|
|
+ db_info_df = db_info_df[db_info_df['DatabaseName'] == 'GC']
|
|
|
+ db_info_df['table'] = db_info_df['TableName'].str.lower()
|
|
|
+ db_info_df.set_index('table', inplace=True)
|
|
|
+ return db_info_df.to_dict(orient='index')
|
|
|
+
|
|
|
+
|
|
|
+def get_fm_sources(base_dir, cfg):
|
|
|
+ bs = BeautifulSoup(open(base_dir + '\\config\\fm\\model.xml', 'r'), 'xml')
|
|
|
+ sources = defaultdict(list)
|
|
|
+ for item in bs.find_all('queryItem'):
|
|
|
+ p = item.parent.parent.find('name').string
|
|
|
+ if item.parent.parent.name == 'folder':
|
|
|
+ p = item.parent.parent.parent.find('name').string
|
|
|
+ parent = "[{0}].[{1}]".format(
|
|
|
+ p,
|
|
|
+ item.parent.find('name').string
|
|
|
+ )
|
|
|
+ src = ''
|
|
|
+ exp = ''
|
|
|
+ if item.expression:
|
|
|
+ if item.expression.refobj:
|
|
|
+ src = item.expression.refobj.string
|
|
|
+ else:
|
|
|
+ exp = item.expression.string
|
|
|
+ elif item.externalName:
|
|
|
+ exp = item.externalName.string
|
|
|
+ sources[parent].append((item.find('name').string, src, exp))
|
|
|
+ interface = {}
|
|
|
+ for k, fields in sources.items():
|
|
|
+ if '[Schnittstelle]' not in k:
|
|
|
+ continue
|
|
|
+ key = k.split('.')[-1][1:-1].lower()
|
|
|
+ links = []
|
|
|
+ for field in fields:
|
|
|
+ links.append(follow_links(sources, field, ''))
|
|
|
+ interface[key] = sorted(list(set([re.search(r'\.\[(.*)\]$', e)[1].lower() for e in links if '[Import]' in e])))
|
|
|
+
|
|
|
+ return interface
|
|
|
+
|
|
|
+
|
|
|
+def follow_links(sources, field, value):
|
|
|
+ if field[1] == '':
|
|
|
+ if field[2] == field[0]:
|
|
|
+ return value
|
|
|
+ return field[2]
|
|
|
+ match = re.search(r'(\[.*\]\.\[.*\])\.\[(.*)\]', field[1])
|
|
|
+ key1 = match[1]
|
|
|
+ val1 = match[2]
|
|
|
+ if key1 in sources:
|
|
|
+ for field2 in sources[key1]:
|
|
|
+ if field2[0] != val1:
|
|
|
+ continue
|
|
|
+ return follow_links(sources, field2, key1)
|
|
|
+ return key1
|
|
|
+
|
|
|
+
|
|
|
+def get_datasources(base_dir, cfg, path_info):
|
|
|
+ all_datasources = set([re.search(r'\\iqd\\.*\\(.*)\.imr', k)[1]
|
|
|
+ for k in path_info.keys()
|
|
|
+ if re.search(r'\\iqd\\.*\\.*\.imr', k)])
|
|
|
+ datasources = {}
|
|
|
+ for ds in all_datasources:
|
|
|
+ ds_search = f'\\{ds}.imr'
|
|
|
+ imr_files = [k for k in path_info.keys()
|
|
|
+ if re.search(r'\\iqd\\.*\.imr', k) and ds_search in k
|
|
|
+ and 'austausch' not in k]
|
|
|
+ if len(imr_files) == 0:
|
|
|
+ imr_file = '0.imr'
|
|
|
+ else:
|
|
|
+ imr_file = imr_files.pop(0)
|
|
|
+ datasources[ds] = {
|
|
|
+ 'imr_file': path_info.get(imr_file, {'mtime': '0'}),
|
|
|
+ 'iqd_file': path_info.get(imr_file[:-4] + '.iqd', {'mtime': '0'}),
|
|
|
+ 'csv_file': path_info.get(cfg.system_dir + '\\export\\' + ds + '.csv', {'mtime': '0'}),
|
|
|
+ 'csv_file_iqd_folder': path_info.get(imr_file[:-4] + '.csv', {'mtime': '0'}),
|
|
|
+ 'duplicates': imr_files
|
|
|
+ }
|
|
|
+ return datasources
|
|
|
+
|
|
|
+
|
|
|
+def cubes_to_models(models):
|
|
|
+ models_sort = sorted([(v.get('logfile_tasks')['mtime'],
|
|
|
+ v.get('logfile_system')['mtime'],
|
|
|
+ v.get('model_file_filled')['mtime'],
|
|
|
+ v.get('model_file')['mtime'],
|
|
|
+ k, v['cube']) for k, v in models.items()])
|
|
|
+ result = {}
|
|
|
+ for m in models_sort:
|
|
|
+ result[m[5]] = m[4]
|
|
|
+ return result
|
|
|
+
|
|
|
+
|
|
|
+def main():
|
|
|
+ base_dir = 'status_server\\unzipped'
|
|
|
+ cfg = Config(str(Path(base_dir + '\\gaps.ini').absolute()))
|
|
|
+ # Dateiliste
|
|
|
+ path_info = get_path_info(base_dir)
|
|
|
+ # random_bat_file = [k for k in path_info.keys() if re.search(r'\\Tasks\\.*\.bat', k)][0]
|
|
|
+ # portal_dir = re.search(r'(.*)\\Tasks\\.*\.bat', random_bat_file)[1]
|
|
|
+ # print(path_info)
|
|
|
+ # Liste aller Cubes
|
|
|
+ result = {}
|
|
|
+
|
|
|
+ # Modelle und Datenquellen
|
|
|
+ result['fm_sources'] = get_fm_sources(base_dir, cfg)
|
|
|
+ result['models'] = get_models(base_dir, cfg, path_info, result['fm_sources'])
|
|
|
+ result['database'] = get_database_info(base_dir, cfg)
|
|
|
+ result['datasources'] = get_datasources(base_dir, cfg, path_info)
|
|
|
+ cubes_models = cubes_to_models(result['models'])
|
|
|
+ result['cubes'] = get_cubes(base_dir, cfg, path_info, cubes_models)
|
|
|
+ # Cubes aktuell?
|
|
|
+ # Rest aktuell?
|
|
|
+ json.dump(result, open('status_server/logs/export.json', 'w'), indent=2)
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == '__main__':
|
|
|
+ main()
|
Global Cube