logging_setup.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265
  1. """Safe structured logging shared by HTTP, database and CLI entrypoints."""
  2. from contextvars import ContextVar
  3. from datetime import datetime, timedelta, timezone
  4. import hashlib
  5. import json
  6. import logging
  7. import os
  8. from pathlib import Path
  9. import re
  10. import sys
  11. import threading
  12. import time
  13. import traceback
  14. from uuid import uuid4
  15. from zoneinfo import ZoneInfo
  16. from concurrent_log_handler import ConcurrentTimedRotatingFileHandler
  17. request_context = ContextVar('request_logging_context', default=None)
  18. db_operation = ContextVar('database_logging_operation', default=None)
  19. db_query = ContextVar('database_logging_query', default=None)
  20. BERLIN = ZoneInfo('Europe/Berlin')
  21. PROJECT_ROOT = Path(__file__).resolve().parent.parent
  22. _lock = threading.RLock()
  23. _configuration = None
  24. _SAFE_FIELDS = ('event', 'operation', 'query_name', 'query_fingerprint', 'sql', 'sql_truncated',
  25. 'sqlstate', 'db_code', 'status', 'duration_ms', 'outcome', 'size_bytes', 'count',
  26. 'reason', 'error_type')
  27. def clean(value):
  28. return str(value).replace('\r', '\\r').replace('\n', '\\n').replace('\x00', '\\0')[:8192]
  29. def sanitize_sql(statement):
  30. """Lex T-SQL literals/comments without modifying quoted identifiers or bind names."""
  31. source, output, i = str(statement), [], 0
  32. while i < len(source):
  33. char = source[i]
  34. if source.startswith('--', i):
  35. end = source.find('\n', i + 2); i = len(source) if end < 0 else end
  36. output.append(' ')
  37. elif source.startswith('/*', i):
  38. depth = 1; i += 2
  39. while i < len(source) and depth:
  40. if source.startswith('/*', i): depth += 1; i += 2
  41. elif source.startswith('*/', i): depth -= 1; i += 2
  42. else: i += 1
  43. output.append(' ')
  44. elif char == "'" or (char in 'nN' and source[i:i+2].endswith("'") and (i == 0 or not source[i-1].isalnum())):
  45. if char != "'": i += 1
  46. i += 1
  47. while i < len(source):
  48. if source[i:i+2] == "''": i += 2
  49. elif source[i] == "'": i += 1; break
  50. else: i += 1
  51. output.append('?')
  52. elif char in '["':
  53. end_char = ']' if char == '[' else '"'
  54. start = i; i += 1
  55. while i < len(source):
  56. if source[i:i+2] == end_char * 2: i += 2
  57. elif source[i] == end_char: i += 1; break
  58. else: i += 1
  59. output.append(source[start:i] if char == '[' else '?')
  60. elif (char.isdecimal() or char == '.' and i+1 < len(source) and source[i+1].isdecimal()) and (i == 0 or not (source[i-1].isalnum() or source[i-1] in '_:@$')):
  61. match = re.match(r'(?:0[xX][0-9a-fA-F]+|(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?)', source[i:])
  62. output.append('?'); i += len(match[0])
  63. else:
  64. output.append(char); i += 1
  65. return ' '.join(''.join(output).split())
  66. def exception_roots(error):
  67. seen = set()
  68. def visit(exc):
  69. if id(exc) in seen: return []
  70. seen.add(id(exc))
  71. if isinstance(exc, BaseExceptionGroup):
  72. return [root for child in exc.exceptions for root in visit(child)]
  73. original = getattr(exc, 'orig', None) or exc.__cause__
  74. return visit(original) if isinstance(original, BaseException) else [exc]
  75. return visit(error)
  76. def frames(tb=None):
  77. extracted = traceback.extract_tb(tb) if tb else traceback.extract_stack()[:-1]
  78. return [dict(file=Path(item.filename).name, line=item.lineno, function=item.name) for item in extracted[-45:]]
  79. class ContextFilter(logging.Filter):
  80. def filter(self, record):
  81. if record.name.startswith('uvicorn.') and record.levelno < logging.INFO: return False
  82. if not record.name.startswith(('app.', 'uvicorn.', '__main__', 'filter_export')) and record.levelno < logging.WARNING: return False
  83. if not hasattr(record, 'event_id'): record.event_id = uuid4().hex
  84. context = request_context.get()
  85. request_id = context['request_id'] if context else None
  86. if record.exc_info and record.exc_info[1]:
  87. roots = exception_roots(record.exc_info[1])
  88. fresh = [exc for exc in roots if getattr(exc, '_gc_log_event', None) in (None, record.event_id)
  89. or request_id is not None and getattr(exc, '_gc_log_request', None) != request_id]
  90. if not fresh: return False
  91. for exc in fresh:
  92. exc._gc_log_event = record.event_id
  93. exc._gc_log_request = request_id
  94. record.safe_exceptions = [dict(type=type(exc).__module__ + '.' + type(exc).__name__,
  95. frames=getattr(exc, '_gc_log_frames', None) or frames(exc.__traceback__)) for exc in fresh]
  96. record.request_fields = {}
  97. if context:
  98. scope = context['scope']
  99. principal = scope.get('state', {}).get('principal')
  100. record.request_fields = dict(request_id=context['request_id'], user=getattr(principal, 'username', None),
  101. method=scope['method'], path=scope.get('path', ''))
  102. return True
  103. def record_data(record):
  104. data = dict(timestamp=datetime.fromtimestamp(record.created, BERLIN).isoformat(timespec='milliseconds'),
  105. level=record.levelname, logger=record.name, event_id=record.event_id,
  106. event=getattr(record, 'event', 'application'),
  107. message=clean(record.getMessage()) if record.name.startswith(('app.', 'uvicorn.', '__main__', 'filter_export')) else 'External logger event')
  108. data.update({key: clean(value) if isinstance(value, str) else value for key, value in record.request_fields.items()})
  109. data.update({key: getattr(record, key) for key in _SAFE_FIELDS if hasattr(record, key)})
  110. if hasattr(record, 'safe_exceptions'): data['exceptions'] = record.safe_exceptions
  111. return data
  112. class JsonFormatter(logging.Formatter):
  113. def format(self, record):
  114. return json.dumps(record_data(record), ensure_ascii=True, separators=(',', ':'), default=lambda _: '[unsupported]')
  115. class ConsoleFormatter(logging.Formatter):
  116. def format(self, record):
  117. data = record_data(record)
  118. suffix = {key: value for key, value in data.items() if key not in ('timestamp', 'level', 'logger', 'message')}
  119. return f"{data['timestamp']} {data['level']} {data['logger']}: {data['message']} {json.dumps(suffix, ensure_ascii=True, separators=(',', ':'))}"
  120. class SafeConsoleHandler(logging.StreamHandler):
  121. def emit(self, record):
  122. super().emit(record)
  123. record._gc_console_emitted = True
  124. def handleError(self, record):
  125. pass # A broken console must not change application behavior or expose raw LogRecords.
  126. class ApplicationFileHandler(ConcurrentTimedRotatingFileHandler):
  127. def __init__(self, filename, retention_days=30):
  128. self.retention_days = retention_days
  129. self.last_fallback = float('-inf')
  130. super().__init__(filename=filename, when='midnight', interval=1, utc=True,
  131. backupCount=retention_days, encoding='utf-8', maxBytes=0)
  132. self.cleanup_archives()
  133. def cleanup_archives(self):
  134. cutoff = datetime.now(timezone.utc).date() - timedelta(days=self.retention_days)
  135. path = Path(self.baseFilename)
  136. for candidate in path.parent.glob(path.name + '.*'):
  137. suffix = candidate.name[len(path.name)+1:]
  138. if not re.fullmatch(r'\d{4}-\d{2}-\d{2}', suffix): continue
  139. try:
  140. if datetime.strptime(suffix, '%Y-%m-%d').date() < cutoff: candidate.unlink(missing_ok=True)
  141. except (OSError, ValueError):
  142. self.fallback(None, 'log_cleanup_failed')
  143. def doRollover(self):
  144. try:
  145. super().doRollover()
  146. except Exception:
  147. self.fallback(None, 'log_rotation_failed')
  148. raise
  149. self.cleanup_archives()
  150. def fallback(self, record, event='log_file_unavailable'):
  151. handler = SafeConsoleHandler(sys.stderr)
  152. handler.setFormatter(ConsoleFormatter())
  153. if time.monotonic() - self.last_fallback >= 60:
  154. self.last_fallback = time.monotonic()
  155. notice = logging.LogRecord('app.logging', logging.ERROR, __file__, 0, 'Logdatei nicht verfügbar; Konsolenfallback aktiv', (), None)
  156. notice.event = event
  157. ContextFilter().filter(notice); handler.handle(notice)
  158. if record is not None and not getattr(record, '_gc_console_emitted', False): handler.handle(record)
  159. def handleError(self, record):
  160. self.fallback(record)
  161. def configure_logging(config=None, *, console_stream=None):
  162. """Replace only our own handlers; repeated startup in one process is a no-op."""
  163. global _configuration
  164. if config is None:
  165. from .config import settings
  166. config = settings
  167. directory = Path(config.LOG_DIR)
  168. if not directory.is_absolute(): directory = PROJECT_ROOT / directory
  169. signature = (os.getpid(), str(directory), config.LOG_LEVEL, config.LOG_TO_FILE, config.LOG_TO_CONSOLE,
  170. config.LOG_RETENTION_DAYS, config.LOG_SQL_STATEMENT, id(console_stream))
  171. with _lock:
  172. root = logging.getLogger()
  173. if _configuration == signature and any(getattr(handler, '_gc_handler', False) for handler in root.handlers): return
  174. for handler in list(root.handlers):
  175. if getattr(handler, '_gc_handler', False): root.removeHandler(handler); handler.close()
  176. handlers, file_failure = [], False
  177. if config.LOG_TO_CONSOLE:
  178. console = SafeConsoleHandler(console_stream or sys.stderr); console.setFormatter(ConsoleFormatter()); handlers.append(console)
  179. if config.LOG_TO_FILE:
  180. try:
  181. directory.mkdir(parents=True, exist_ok=True)
  182. file_handler = ApplicationFileHandler(str(directory / 'application.log'), config.LOG_RETENTION_DAYS)
  183. file_handler.setFormatter(JsonFormatter()); handlers.append(file_handler)
  184. except Exception:
  185. file_failure = True
  186. if not handlers:
  187. console = SafeConsoleHandler(console_stream or sys.stderr); console.setFormatter(ConsoleFormatter()); handlers.append(console)
  188. if not handlers: handlers.append(logging.NullHandler())
  189. for handler in handlers:
  190. handler._gc_handler = True; handler.addFilter(ContextFilter()); root.addHandler(handler)
  191. root.setLevel(config.LOG_LEVEL)
  192. for name in ('uvicorn', 'uvicorn.error', 'uvicorn.access'):
  193. logger = logging.getLogger(name); logger.handlers.clear(); logger.propagate = True
  194. logger.setLevel(logging.INFO)
  195. logging.getLogger('uvicorn.access').disabled = True
  196. for name in ('sqlalchemy', 'urllib3', 'httpx', 'httpcore', 'ldap3', 'docuware'):
  197. logging.getLogger(name).setLevel(logging.WARNING)
  198. _configuration = signature
  199. if file_failure: logging.getLogger('app.logging').error('Logverzeichnis nicht verfügbar; Konsolenfallback aktiv', extra={'event': 'log_file_unavailable'})
  200. def log_query_error(error, statement=None, query_name=None, operation=None):
  201. from .config import settings
  202. try:
  203. sql = sanitize_sql(statement) if statement is not None else ''
  204. except Exception:
  205. sql = '[statement unavailable]'
  206. metadata = dict(event='database_error', operation=operation or db_operation.get() or 'execute',
  207. query_name=query_name or query_origin(), query_fingerprint=hashlib.sha256(sql.encode()).hexdigest())
  208. if settings.LOG_SQL_STATEMENT and sql:
  209. metadata.update(sql=sql[:8192], sql_truncated=len(sql) > 8192)
  210. roots = exception_roots(error)
  211. for root in roots:
  212. if not getattr(root, '_gc_log_frames', None): root._gc_log_frames = frames()
  213. first = root.args[0] if root.args else None
  214. if isinstance(first, str) and re.fullmatch(r'[A-Z0-9]{5}', first): metadata['sqlstate'] = first
  215. code = getattr(root, 'sqlite_errorcode', None)
  216. if isinstance(code, int): metadata['db_code'] = code
  217. logging.getLogger('app.db').error('Datenbankoperation fehlgeschlagen', exc_info=(type(error), error, error.__traceback__), extra=metadata)
  218. def query_origin():
  219. for item in reversed(traceback.extract_stack()[:-1]):
  220. path = Path(item.filename)
  221. if PROJECT_ROOT / 'app' in path.parents and path.name not in ('logging_setup.py', 'db.py'):
  222. return f'{path.name}:{item.name}'
  223. return 'database'
  224. def run_cli(callback):
  225. configure_logging()
  226. try:
  227. callback()
  228. except Exception:
  229. logging.getLogger('app.cli').exception('Hilfsprogramm fehlgeschlagen', extra={'event': 'cli_failed'})
  230. raise SystemExit(1) from None