file_storage.py 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228
  1. """Document storage, including legacy folders and collision-safe uploads."""
  2. from contextlib import contextmanager
  3. from dataclasses import dataclass
  4. from datetime import datetime
  5. import hashlib
  6. import logging
  7. import os
  8. from pathlib import Path
  9. import re
  10. import tempfile
  11. import threading
  12. import time
  13. from weakref import WeakValueDictionary
  14. from zoneinfo import ZoneInfo
  15. from .config import settings
  16. logger = logging.getLogger(__name__)
  17. BERLIN = ZoneInfo('Europe/Berlin')
  18. _THREAD_LOCKS = WeakValueDictionary()
  19. _LOCK_GUARD = threading.Lock()
  20. _RESERVED = {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | {
  21. f'{prefix}{digit}' for prefix in ('COM', 'LPT') for digit in '123456789¹²³'
  22. }
  23. class UploadError(ValueError):
  24. """An actionable error which can be displayed next to an individual file."""
  25. @dataclass(frozen=True)
  26. class StoredFile:
  27. name: str
  28. relative_path: str
  29. modified_at: datetime
  30. size: int
  31. def validate_filename(name: str) -> str:
  32. if (not name or name in {'.', '..'} or name.endswith(('.', ' '))
  33. or re.search(r'[<>:"/\\|?*\x00-\x1f\x7f]', name)
  34. or name.split('.')[0].rstrip(' ').upper() in _RESERVED
  35. or len(name.encode('utf-16-le', errors='surrogatepass')) // 2 > 255):
  36. raise UploadError('Dieser Dateiname kann unter Windows nicht gespeichert werden.')
  37. return name
  38. def _root() -> Path:
  39. if not settings.LOCAL_STORAGE:
  40. raise UploadError('Die Dateiablage ist nicht konfiguriert.')
  41. return Path(settings.LOCAL_STORAGE).resolve()
  42. def _checked(root: Path, *parts: str) -> Path:
  43. for part in parts:
  44. validate_filename(part)
  45. candidate = root.joinpath(*parts)
  46. resolved = candidate.resolve()
  47. if root not in resolved.parents or resolved != candidate:
  48. raise UploadError('Der Ablagepfad ist nicht zulässig.')
  49. return candidate
  50. def _folders(root: Path, key: dict[str, str]) -> tuple[Path, Path]:
  51. tenant, number = str(key['Client_DB']), str(key['Beleg_Nr'])
  52. validate_filename(tenant)
  53. validate_filename(number)
  54. return _checked(root, f'{tenant}_{number}'), _checked(root, tenant, number)
  55. def file_primary_key(name: str) -> dict[str, str]:
  56. # Query parameters are already URL-decoded by the framework. Do not decode twice.
  57. parts = name.replace('\\', '/').split('/')
  58. if len(parts) not in (2, 3):
  59. raise FileNotFoundError(name)
  60. try:
  61. for part in parts:
  62. validate_filename(part)
  63. if len(parts) == 2:
  64. tenant, separator, number = parts[0].partition('_')
  65. if not separator or not tenant or not number:
  66. raise FileNotFoundError(name)
  67. else:
  68. tenant, number = parts[:2]
  69. except UploadError as error:
  70. raise FileNotFoundError(name) from error
  71. return {'Client_DB': tenant, 'Beleg_Nr': number}
  72. def get_files(primary_key: dict[str, str]) -> list[StoredFile]:
  73. if not settings.LOCAL_STORAGE:
  74. return []
  75. root = _root()
  76. try:
  77. folders = _folders(root, primary_key)
  78. except UploadError:
  79. return []
  80. files = []
  81. for folder in folders:
  82. for entry in folder.glob('*'):
  83. try:
  84. filename = _checked(root, *entry.relative_to(root).parts)
  85. if not filename.is_file():
  86. continue
  87. stat = filename.stat()
  88. files.append(StoredFile(filename.name, filename.relative_to(root).as_posix(),
  89. datetime.fromtimestamp(stat.st_mtime, BERLIN), stat.st_size))
  90. except (FileNotFoundError, UploadError):
  91. continue
  92. return sorted(files, key=lambda item: (-item.modified_at.timestamp(), item.name.casefold(), item.relative_path))
  93. def get_file(name: str) -> Path:
  94. if not settings.LOCAL_STORAGE:
  95. raise FileNotFoundError(name)
  96. file_primary_key(name)
  97. try:
  98. filename = _checked(_root(), *name.replace('\\', '/').split('/'))
  99. if not filename.is_file():
  100. raise FileNotFoundError(name)
  101. return filename
  102. except UploadError as error:
  103. raise FileNotFoundError(name) from error
  104. @contextmanager
  105. def _beleg_lock(root: Path, folder: Path):
  106. lock_dir = _checked(root, '.upload-locks')
  107. lock_dir.mkdir(parents=True, exist_ok=True)
  108. digest = hashlib.sha256(str(folder).casefold().encode('utf-8')).hexdigest()
  109. lock_path = _checked(root, '.upload-locks', digest + '.lock')
  110. with _LOCK_GUARD:
  111. lock = _THREAD_LOCKS.setdefault(str(lock_path), threading.Lock())
  112. deadline = time.monotonic() + 30
  113. if not lock.acquire(timeout=30):
  114. raise UploadError('Die Dateiablage ist gerade belegt. Bitte erneut abschicken.')
  115. try:
  116. with lock_path.open('a+b') as handle:
  117. if handle.tell() == 0:
  118. handle.write(b'0')
  119. handle.flush()
  120. locked = False
  121. try:
  122. while not locked:
  123. try:
  124. handle.seek(0)
  125. if os.name == 'nt':
  126. import msvcrt
  127. msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
  128. else:
  129. import fcntl
  130. fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
  131. locked = True
  132. except OSError:
  133. if time.monotonic() >= deadline:
  134. raise UploadError('Die Dateiablage ist gerade belegt. Bitte erneut abschicken.')
  135. time.sleep(.05)
  136. yield
  137. finally:
  138. if locked:
  139. handle.seek(0)
  140. if os.name == 'nt':
  141. import msvcrt
  142. msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
  143. else:
  144. import fcntl
  145. fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
  146. finally:
  147. lock.release()
  148. def _archive_name(root: Path, original: Path) -> Path:
  149. stamp = datetime.now(BERLIN).strftime('%Y%m%d_%H%M%S_%f')
  150. counter = 0
  151. while True:
  152. suffix = f'_{counter}' if counter else ''
  153. name = f'{original.stem}_{stamp}{suffix}{original.suffix}'
  154. archive = _checked(root, *original.parent.relative_to(root).parts, name)
  155. if not archive.exists():
  156. return archive
  157. counter += 1
  158. def save_upload(primary_key: dict[str, str], name: str, stream, max_bytes: int) -> str:
  159. """Stage and finalize one file; keep earlier successful files independent."""
  160. validate_filename(name)
  161. root = _root()
  162. destination, legacy = _folders(root, primary_key)
  163. staging = _checked(root, '.upload-staging')
  164. staging.mkdir(parents=True, exist_ok=True)
  165. temporary = None
  166. try:
  167. with tempfile.NamedTemporaryFile(dir=staging, suffix='.tmp', delete=False) as output:
  168. temporary = Path(output.name)
  169. stream.seek(0)
  170. size = 0
  171. while chunk := stream.read(1024 * 1024):
  172. size += len(chunk)
  173. if size > max_bytes:
  174. raise UploadError(f'Die Datei überschreitet die erlaubten {settings.UPLOAD_MAX_FILE_SIZE_MB} MB.')
  175. output.write(chunk)
  176. output.flush()
  177. os.fsync(output.fileno())
  178. with _beleg_lock(root, destination):
  179. destination.mkdir(parents=True, exist_ok=True)
  180. target = _checked(root, *destination.relative_to(root).parts, name)
  181. previous = []
  182. try:
  183. for folder in (destination, legacy):
  184. existing = _checked(root, *folder.relative_to(root).parts, name)
  185. if existing.exists():
  186. if not existing.is_file():
  187. raise UploadError('Unter diesem Dateinamen besteht bereits ein Ordner.')
  188. archive = _archive_name(root, existing)
  189. existing.rename(archive)
  190. previous.append((existing, archive))
  191. os.replace(temporary, target)
  192. except Exception:
  193. for existing, archive in reversed(previous):
  194. try:
  195. archive.rename(existing)
  196. except OSError:
  197. logger.exception('Archivierte Datei konnte nicht zurückbenannt werden: %s', archive)
  198. raise
  199. return target.relative_to(root).as_posix()
  200. finally:
  201. if temporary and temporary.exists():
  202. temporary.unlink()