| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246 |
- """Document storage, including legacy folders and collision-safe uploads."""
- import hashlib
- import logging
- import os
- import re
- import tempfile
- import threading
- import time
- from contextlib import contextmanager
- from dataclasses import dataclass
- from datetime import datetime
- from pathlib import Path
- from weakref import WeakValueDictionary
- from zoneinfo import ZoneInfo
- from .config import settings
- logger = logging.getLogger(__name__)
- BERLIN = ZoneInfo("Europe/Berlin")
- _THREAD_LOCKS = WeakValueDictionary()
- _LOCK_GUARD = threading.Lock()
- _RESERVED = {"CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$"} | {
- f"{prefix}{digit}" for prefix in ("COM", "LPT") for digit in "123456789¹²³"
- }
- class UploadError(ValueError):
- """An actionable error which can be displayed next to an individual file."""
- @dataclass(frozen=True)
- class StoredFile:
- name: str
- relative_path: str
- modified_at: datetime
- size: int
- def validate_filename(name: str) -> str:
- if (
- not name
- or name in {".", ".."}
- or name.endswith((".", " "))
- or re.search(r'[<>:"/\\|?*\x00-\x1f\x7f]', name)
- or name.split(".")[0].rstrip(" ").upper() in _RESERVED
- or len(name.encode("utf-16-le", errors="surrogatepass")) // 2 > 255
- ):
- raise UploadError("Dieser Dateiname kann unter Windows nicht gespeichert werden.")
- return name
- def _root() -> Path:
- if not settings.LOCAL_STORAGE:
- raise UploadError("Die Dateiablage ist nicht konfiguriert.")
- return Path(settings.LOCAL_STORAGE).resolve()
- def _checked(root: Path, *parts: str) -> Path:
- for part in parts:
- validate_filename(part)
- candidate = root.joinpath(*parts)
- resolved = candidate.resolve()
- if root not in resolved.parents or resolved != candidate:
- raise UploadError("Der Ablagepfad ist nicht zulässig.")
- return candidate
- def _folders(root: Path, key: dict[str, str]) -> tuple[Path, Path]:
- tenant, number = str(key["Client_DB"]), str(key["Beleg_Nr"])
- validate_filename(tenant)
- validate_filename(number)
- return _checked(root, f"{tenant}_{number}"), _checked(root, tenant, number)
- def file_primary_key(name: str) -> dict[str, str]:
- # Query parameters are already URL-decoded by the framework. Do not decode twice.
- parts = name.replace("\\", "/").split("/")
- if len(parts) not in (2, 3):
- raise FileNotFoundError(name)
- try:
- for part in parts:
- validate_filename(part)
- if len(parts) == 2:
- tenant, separator, number = parts[0].partition("_")
- if not separator or not tenant or not number:
- raise FileNotFoundError(name)
- else:
- tenant, number = parts[:2]
- except UploadError as error:
- raise FileNotFoundError(name) from error
- return {"Client_DB": tenant, "Beleg_Nr": number}
- def get_files(primary_key: dict[str, str]) -> list[StoredFile]:
- if not settings.LOCAL_STORAGE:
- return []
- root = _root()
- try:
- folders = _folders(root, primary_key)
- except UploadError:
- return []
- files = []
- for folder in folders:
- for entry in folder.glob("*"):
- try:
- filename = _checked(root, *entry.relative_to(root).parts)
- if not filename.is_file():
- continue
- stat = filename.stat()
- files.append(
- StoredFile(
- filename.name,
- filename.relative_to(root).as_posix(),
- datetime.fromtimestamp(stat.st_mtime, BERLIN),
- stat.st_size,
- )
- )
- except (FileNotFoundError, UploadError):
- continue
- return sorted(files, key=lambda item: (-item.modified_at.timestamp(), item.name.casefold(), item.relative_path))
- def get_file(name: str) -> Path:
- if not settings.LOCAL_STORAGE:
- raise FileNotFoundError(name)
- file_primary_key(name)
- try:
- filename = _checked(_root(), *name.replace("\\", "/").split("/"))
- if not filename.is_file():
- raise FileNotFoundError(name)
- return filename
- except UploadError as error:
- raise FileNotFoundError(name) from error
- @contextmanager
- def _beleg_lock(root: Path, folder: Path):
- lock_dir = _checked(root, ".upload-locks")
- lock_dir.mkdir(parents=True, exist_ok=True)
- digest = hashlib.sha256(str(folder).casefold().encode("utf-8")).hexdigest()
- lock_path = _checked(root, ".upload-locks", digest + ".lock")
- with _LOCK_GUARD:
- lock = _THREAD_LOCKS.setdefault(str(lock_path), threading.Lock())
- deadline = time.monotonic() + 30
- if not lock.acquire(timeout=30):
- raise UploadError("Die Dateiablage ist gerade belegt. Bitte erneut abschicken.")
- try:
- with lock_path.open("a+b") as handle:
- if handle.tell() == 0:
- handle.write(b"0")
- handle.flush()
- locked = False
- try:
- while not locked:
- try:
- handle.seek(0)
- if os.name == "nt":
- import msvcrt
- msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
- else:
- import fcntl
- fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
- locked = True
- except OSError:
- if time.monotonic() >= deadline:
- raise UploadError("Die Dateiablage ist gerade belegt. Bitte erneut abschicken.")
- time.sleep(0.05)
- yield
- finally:
- if locked:
- handle.seek(0)
- if os.name == "nt":
- import msvcrt
- msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
- else:
- import fcntl
- fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
- finally:
- lock.release()
- def _archive_name(root: Path, original: Path) -> Path:
- stamp = datetime.now(BERLIN).strftime("%Y%m%d_%H%M%S_%f")
- counter = 0
- while True:
- suffix = f"_{counter}" if counter else ""
- name = f"{original.stem}_{stamp}{suffix}{original.suffix}"
- archive = _checked(root, *original.parent.relative_to(root).parts, name)
- if not archive.exists():
- return archive
- counter += 1
- def save_upload(primary_key: dict[str, str], name: str, stream, max_bytes: int) -> str:
- """Stage and finalize one file; keep earlier successful files independent."""
- validate_filename(name)
- root = _root()
- destination, legacy = _folders(root, primary_key)
- staging = _checked(root, ".upload-staging")
- staging.mkdir(parents=True, exist_ok=True)
- temporary = None
- try:
- with tempfile.NamedTemporaryFile(dir=staging, suffix=".tmp", delete=False) as output:
- temporary = Path(output.name)
- stream.seek(0)
- size = 0
- while chunk := stream.read(1024 * 1024):
- size += len(chunk)
- if size > max_bytes:
- raise UploadError(f"Die Datei überschreitet die erlaubten {settings.UPLOAD_MAX_FILE_SIZE_MB} MB.")
- output.write(chunk)
- output.flush()
- os.fsync(output.fileno())
- with _beleg_lock(root, destination):
- destination.mkdir(parents=True, exist_ok=True)
- target = _checked(root, *destination.relative_to(root).parts, name)
- previous = []
- try:
- for folder in (destination, legacy):
- existing = _checked(root, *folder.relative_to(root).parts, name)
- if existing.exists():
- if not existing.is_file():
- raise UploadError("Unter diesem Dateinamen besteht bereits ein Ordner.")
- archive = _archive_name(root, existing)
- existing.rename(archive)
- previous.append((existing, archive))
- os.replace(temporary, target)
- except Exception:
- for existing, archive in reversed(previous):
- try:
- archive.rename(existing)
- except OSError:
- logger.exception(
- "Archivierte Datei konnte nicht zurückbenannt werden",
- extra={"event": "file_archive_rollback_failed"},
- )
- raise
- return target.relative_to(root).as_posix()
- finally:
- if temporary and temporary.exists():
- temporary.unlink()
|