file_storage.py 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246
  1. """Document storage, including legacy folders and collision-safe uploads."""
  2. import hashlib
  3. import logging
  4. import os
  5. import re
  6. import tempfile
  7. import threading
  8. import time
  9. from contextlib import contextmanager
  10. from dataclasses import dataclass
  11. from datetime import datetime
  12. from pathlib import Path
  13. from weakref import WeakValueDictionary
  14. from zoneinfo import ZoneInfo
  15. from .config import settings
  16. logger = logging.getLogger(__name__)
  17. BERLIN = ZoneInfo("Europe/Berlin")
  18. _THREAD_LOCKS = WeakValueDictionary()
  19. _LOCK_GUARD = threading.Lock()
  20. _RESERVED = {"CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$"} | {
  21. f"{prefix}{digit}" for prefix in ("COM", "LPT") for digit in "123456789¹²³"
  22. }
  23. class UploadError(ValueError):
  24. """An actionable error which can be displayed next to an individual file."""
  25. @dataclass(frozen=True)
  26. class StoredFile:
  27. name: str
  28. relative_path: str
  29. modified_at: datetime
  30. size: int
  31. def validate_filename(name: str) -> str:
  32. if (
  33. not name
  34. or name in {".", ".."}
  35. or name.endswith((".", " "))
  36. or re.search(r'[<>:"/\\|?*\x00-\x1f\x7f]', name)
  37. or name.split(".")[0].rstrip(" ").upper() in _RESERVED
  38. or len(name.encode("utf-16-le", errors="surrogatepass")) // 2 > 255
  39. ):
  40. raise UploadError("Dieser Dateiname kann unter Windows nicht gespeichert werden.")
  41. return name
  42. def _root() -> Path:
  43. if not settings.LOCAL_STORAGE:
  44. raise UploadError("Die Dateiablage ist nicht konfiguriert.")
  45. return Path(settings.LOCAL_STORAGE).resolve()
  46. def _checked(root: Path, *parts: str) -> Path:
  47. for part in parts:
  48. validate_filename(part)
  49. candidate = root.joinpath(*parts)
  50. resolved = candidate.resolve()
  51. if root not in resolved.parents or resolved != candidate:
  52. raise UploadError("Der Ablagepfad ist nicht zulässig.")
  53. return candidate
  54. def _folders(root: Path, key: dict[str, str]) -> tuple[Path, Path]:
  55. tenant, number = str(key["Client_DB"]), str(key["Beleg_Nr"])
  56. validate_filename(tenant)
  57. validate_filename(number)
  58. return _checked(root, f"{tenant}_{number}"), _checked(root, tenant, number)
  59. def file_primary_key(name: str) -> dict[str, str]:
  60. # Query parameters are already URL-decoded by the framework. Do not decode twice.
  61. parts = name.replace("\\", "/").split("/")
  62. if len(parts) not in (2, 3):
  63. raise FileNotFoundError(name)
  64. try:
  65. for part in parts:
  66. validate_filename(part)
  67. if len(parts) == 2:
  68. tenant, separator, number = parts[0].partition("_")
  69. if not separator or not tenant or not number:
  70. raise FileNotFoundError(name)
  71. else:
  72. tenant, number = parts[:2]
  73. except UploadError as error:
  74. raise FileNotFoundError(name) from error
  75. return {"Client_DB": tenant, "Beleg_Nr": number}
  76. def get_files(primary_key: dict[str, str]) -> list[StoredFile]:
  77. if not settings.LOCAL_STORAGE:
  78. return []
  79. root = _root()
  80. try:
  81. folders = _folders(root, primary_key)
  82. except UploadError:
  83. return []
  84. files = []
  85. for folder in folders:
  86. for entry in folder.glob("*"):
  87. try:
  88. filename = _checked(root, *entry.relative_to(root).parts)
  89. if not filename.is_file():
  90. continue
  91. stat = filename.stat()
  92. files.append(
  93. StoredFile(
  94. filename.name,
  95. filename.relative_to(root).as_posix(),
  96. datetime.fromtimestamp(stat.st_mtime, BERLIN),
  97. stat.st_size,
  98. )
  99. )
  100. except (FileNotFoundError, UploadError):
  101. continue
  102. return sorted(files, key=lambda item: (-item.modified_at.timestamp(), item.name.casefold(), item.relative_path))
  103. def get_file(name: str) -> Path:
  104. if not settings.LOCAL_STORAGE:
  105. raise FileNotFoundError(name)
  106. file_primary_key(name)
  107. try:
  108. filename = _checked(_root(), *name.replace("\\", "/").split("/"))
  109. if not filename.is_file():
  110. raise FileNotFoundError(name)
  111. return filename
  112. except UploadError as error:
  113. raise FileNotFoundError(name) from error
  114. @contextmanager
  115. def _beleg_lock(root: Path, folder: Path):
  116. lock_dir = _checked(root, ".upload-locks")
  117. lock_dir.mkdir(parents=True, exist_ok=True)
  118. digest = hashlib.sha256(str(folder).casefold().encode("utf-8")).hexdigest()
  119. lock_path = _checked(root, ".upload-locks", digest + ".lock")
  120. with _LOCK_GUARD:
  121. lock = _THREAD_LOCKS.setdefault(str(lock_path), threading.Lock())
  122. deadline = time.monotonic() + 30
  123. if not lock.acquire(timeout=30):
  124. raise UploadError("Die Dateiablage ist gerade belegt. Bitte erneut abschicken.")
  125. try:
  126. with lock_path.open("a+b") as handle:
  127. if handle.tell() == 0:
  128. handle.write(b"0")
  129. handle.flush()
  130. locked = False
  131. try:
  132. while not locked:
  133. try:
  134. handle.seek(0)
  135. if os.name == "nt":
  136. import msvcrt
  137. msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
  138. else:
  139. import fcntl
  140. fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
  141. locked = True
  142. except OSError:
  143. if time.monotonic() >= deadline:
  144. raise UploadError("Die Dateiablage ist gerade belegt. Bitte erneut abschicken.")
  145. time.sleep(0.05)
  146. yield
  147. finally:
  148. if locked:
  149. handle.seek(0)
  150. if os.name == "nt":
  151. import msvcrt
  152. msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
  153. else:
  154. import fcntl
  155. fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
  156. finally:
  157. lock.release()
  158. def _archive_name(root: Path, original: Path) -> Path:
  159. stamp = datetime.now(BERLIN).strftime("%Y%m%d_%H%M%S_%f")
  160. counter = 0
  161. while True:
  162. suffix = f"_{counter}" if counter else ""
  163. name = f"{original.stem}_{stamp}{suffix}{original.suffix}"
  164. archive = _checked(root, *original.parent.relative_to(root).parts, name)
  165. if not archive.exists():
  166. return archive
  167. counter += 1
  168. def save_upload(primary_key: dict[str, str], name: str, stream, max_bytes: int) -> str:
  169. """Stage and finalize one file; keep earlier successful files independent."""
  170. validate_filename(name)
  171. root = _root()
  172. destination, legacy = _folders(root, primary_key)
  173. staging = _checked(root, ".upload-staging")
  174. staging.mkdir(parents=True, exist_ok=True)
  175. temporary = None
  176. try:
  177. with tempfile.NamedTemporaryFile(dir=staging, suffix=".tmp", delete=False) as output:
  178. temporary = Path(output.name)
  179. stream.seek(0)
  180. size = 0
  181. while chunk := stream.read(1024 * 1024):
  182. size += len(chunk)
  183. if size > max_bytes:
  184. raise UploadError(f"Die Datei überschreitet die erlaubten {settings.UPLOAD_MAX_FILE_SIZE_MB} MB.")
  185. output.write(chunk)
  186. output.flush()
  187. os.fsync(output.fileno())
  188. with _beleg_lock(root, destination):
  189. destination.mkdir(parents=True, exist_ok=True)
  190. target = _checked(root, *destination.relative_to(root).parts, name)
  191. previous = []
  192. try:
  193. for folder in (destination, legacy):
  194. existing = _checked(root, *folder.relative_to(root).parts, name)
  195. if existing.exists():
  196. if not existing.is_file():
  197. raise UploadError("Unter diesem Dateinamen besteht bereits ein Ordner.")
  198. archive = _archive_name(root, existing)
  199. existing.rename(archive)
  200. previous.append((existing, archive))
  201. os.replace(temporary, target)
  202. except Exception:
  203. for existing, archive in reversed(previous):
  204. try:
  205. archive.rename(existing)
  206. except OSError:
  207. logger.exception(
  208. "Archivierte Datei konnte nicht zurückbenannt werden",
  209. extra={"event": "file_archive_rollback_failed"},
  210. )
  211. raise
  212. return target.relative_to(root).as_posix()
  213. finally:
  214. if temporary and temporary.exists():
  215. temporary.unlink()