Coverage for downtify/library_trash.py: 89%
572 statements
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
« prev ^ index » next coverage.py v7.17.4, created at 2026-11-21 22:18 +0000
1"""Persistent, reversible music trash.
3Filesystem work runs in a worker: inside the app (``embedded``, the
4default) or as a separate process (``python -m downtify.trash_worker``)
5for installs whose app only mounts the music read-only. No playlist,
6favourite or ownership record is deleted by this module. A confirmed
7operation reserves its original path until restored or expired.
8"""
10from __future__ import annotations
12import ctypes
13import errno
14import hashlib
15import json
16import os
17import platform
18import re
19import secrets
20import stat
21import sys
22import time
23import unicodedata
24from contextlib import contextmanager
25from dataclasses import dataclass
26from pathlib import Path, PurePosixPath
27from typing import Callable, Mapping, Optional
29if os.name == 'nt':
30 import msvcrt
31else:
32 import fcntl
34from downtify.db.config import Dialect
35from downtify.db.core import Connection, Database, as_database
37TRASH_DIR = '.downtify-trash'
38RETENTION_DAYS = 30
39RETENTION_SECONDS = RETENTION_DAYS * 86400
40PREVIEW_SECONDS = 600
41WORKER_MODES = ('embedded', 'external')
42#: How long the hidden paths are trusted before the journal's revision is
43#: read again (a separate worker process changes it behind our back).
44HIDDEN_RECHECK_SECONDS = 2.0
45_TRUE = {'1', 'true', 'yes', 'on'}
46#: The operations a worker pass picks up (``?`` is the current time).
47_WORK = (
48 "FROM library_trash WHERE state IN ('pending','moving',"
49 "'restore_pending','restoring','purging') OR "
50 "(state='trashed' AND expires<=?)"
51)
52BLOCKING_STATES = (
53 'pending',
54 'moving',
55 'trashed',
56 'restore_pending',
57 'restoring',
58 'purging',
59 'purged',
60 'error',
61)
64_TRASH_COLUMNS = (
65 'id',
66 'batch',
67 'user_id',
68 'root_key',
69 'relative',
70 'stored',
71 'track_id',
72 'row_json',
73 'manifest',
74 'state',
75 'created',
76)
79class TrashError(ValueError):
80 """A stable, translatable failure code; never exposes private paths."""
83def identity_keys(song: dict) -> set[str]:
84 """Use provider IDs and exact release tags, preserving edition names."""
85 keys = set()
86 for field in ('spotify_id', 'deezer_id', 'youtube_id', 'isrc'):
87 value = song.get(field)
88 if value:
89 keys.add(
90 field
91 + ':'
92 + (
93 str(value).strip().upper()
94 if field == 'isrc'
95 else str(value).strip()
96 )
97 )
98 song_id = str(song.get('song_id') or '')
99 if re.fullmatch('[a-zA-Z0-9]{22}', song_id) and song.get('source') not in {
100 'deezer',
101 'youtube',
102 }:
103 keys.add('spotify_id:' + song_id)
104 if song.get('source') == 'youtube' and song_id:
105 keys.add('youtube_id:' + song_id)
107 def normal(value):
108 return ' '.join(
109 unicodedata.normalize('NFKC', str(value or '')).casefold().split()
110 )
112 title = normal(song.get('title') or song.get('name'))
113 album = normal(song.get('album') or song.get('album_name'))
114 artists = song.get('artists')
115 if not isinstance(artists, list) or not artists:
116 artists = str(song.get('artist') or '').split(';')
117 artist = json.dumps(sorted(normal(a) for a in artists if normal(a)))
118 if artist == '[]':
119 artist = ''
120 if title and artist and album:
121 keys.add(
122 'release:'
123 + hashlib.sha256(
124 json.dumps([title, artist, album]).encode()
125 ).hexdigest()
126 )
127 return keys
130@contextmanager
131def worker_lock(path: Path):
132 """Process lifetime lock: a second worker must never race the first."""
133 with path.open('a+b') as handle:
134 if os.name == 'nt':
135 if os.fstat(handle.fileno()).st_size == 0:
136 handle.write(b'0')
137 handle.flush()
138 handle.seek(0)
139 try:
140 msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
141 except OSError:
142 yield False
143 return
144 try:
145 yield True
146 finally:
147 handle.seek(0)
148 msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
149 else:
150 try:
151 fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
152 except BlockingIOError:
153 yield False
154 return
155 try:
156 yield True
157 finally:
158 fcntl.flock(handle, fcntl.LOCK_UN)
161@dataclass(frozen=True)
162class TrashRoot:
163 path: Path
164 marker: str = ''
165 marker_value: str = ''
167 def check(self) -> Path:
168 root = self.path.absolute()
169 if root.is_symlink() or not root.is_dir():
170 raise TrashError('root_unavailable')
171 if self.marker:
172 marker = root / self.marker
173 if marker.is_symlink() or not marker.is_file():
174 raise TrashError('nas_marker_missing')
175 if marker.read_text().strip() != self.marker_value:
176 raise TrashError('nas_marker_changed')
177 # The root itself may not be a link, but a folder above it may
178 # (``/home`` on some distributions): every later step opens each
179 # component without following links, so start from the real path.
180 return root.resolve()
183def _relative(value: str, *, internal: bool = False) -> str:
184 if not value or '\\' in value or '\0' in value:
185 raise TrashError('invalid_path')
186 path = PurePosixPath(value)
187 if path.is_absolute() or any(
188 p in {'', '.', '..'} or ':' in p for p in path.parts
189 ):
190 raise TrashError('invalid_path')
191 if not internal and TRASH_DIR in path.parts:
192 raise TrashError('invalid_path')
193 return path.as_posix()
196def _safe(root: Path, relative: str, *, internal: bool = False) -> Path:
197 relative = _relative(relative, internal=internal)
198 full = root
199 for component in PurePosixPath(relative).parts:
200 full /= component
201 if full.is_symlink():
202 raise TrashError('unsafe_symlink')
203 if not full.resolve().is_relative_to(root.resolve()):
204 raise TrashError('invalid_path')
205 return full
208def fingerprint(path: Path) -> dict:
209 flags = (
210 os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_BINARY', 0)
211 )
212 if os.name == 'nt':
213 fd = os.open(path, flags)
214 else:
215 with _directory_fd(path.parent) as directory:
216 fd = os.open(path.name, flags, dir_fd=directory)
217 try:
218 before = os.fstat(fd)
219 if not stat.S_ISREG(before.st_mode):
220 raise TrashError('not_regular_file')
221 digest = hashlib.sha256()
222 while chunk := os.read(fd, 1024 * 1024):
223 digest.update(chunk)
224 after = os.fstat(fd)
225 keys = ('st_dev', 'st_ino', 'st_size', 'st_mtime_ns', 'st_ctime_ns')
226 if any(getattr(before, k) != getattr(after, k) for k in keys):
227 raise TrashError('file_changed')
228 return {
229 'size': after.st_size,
230 'mtime_ns': after.st_mtime_ns,
231 'dev': after.st_dev,
232 'ino': after.st_ino,
233 'sha256': digest.hexdigest(),
234 }
235 finally:
236 os.close(fd)
239def _matches(path: Path, expected: dict, *, original: bool = False) -> bool:
240 actual = fingerprint(path)
241 keys = (
242 ('size', 'sha256', 'mtime_ns', 'dev', 'ino')
243 if original
244 else ('size', 'sha256')
245 )
246 return all(actual[k] == expected[k] for k in keys)
249def _fsync_dir(path: Path) -> None:
250 if os.name == 'nt':
251 return
252 with _directory_fd(path) as fd:
253 os.fsync(fd)
256#: ``renameat2``'s syscall number, for a C library without the wrapper
257#: (musl before 1.2.5, as in some Alpine images).
258_RENAMEAT2_SYSCALL = {
259 'x86_64': 316,
260 'amd64': 316,
261 'i386': 353,
262 'i686': 353,
263 'aarch64': 276,
264 'arm64': 276,
265 'riscv64': 276,
266 'armv7l': 382,
267 'armv6l': 382,
268}
269_RENAME_NOREPLACE = 1
272def _renameat2() -> Callable[[int, bytes, int, bytes, int], int]:
273 libc = ctypes.CDLL(None, use_errno=True)
274 rename = getattr(libc, 'renameat2', None)
275 if rename is not None:
276 rename.argtypes = [
277 ctypes.c_int,
278 ctypes.c_char_p,
279 ctypes.c_int,
280 ctypes.c_char_p,
281 ctypes.c_uint,
282 ]
283 rename.restype = ctypes.c_int
284 return rename
285 number = _RENAMEAT2_SYSCALL.get(platform.machine().lower())
286 if number is None:
287 raise TrashError('atomic_move_unsupported')
288 syscall = libc.syscall
289 syscall.restype = ctypes.c_long
291 def via_syscall(src_fd, src, dst_fd, dst, flags):
292 return syscall(
293 ctypes.c_long(number),
294 ctypes.c_int(src_fd),
295 ctypes.c_char_p(src),
296 ctypes.c_int(dst_fd),
297 ctypes.c_char_p(dst),
298 ctypes.c_uint(flags),
299 )
301 return via_syscall
304def _rename_no_replace(source: Path, target: Path) -> None:
305 """Same-mount atomic rename which never replaces a concurrent file."""
306 if sys.platform == 'linux':
307 rename = _renameat2()
308 with (
309 _directory_fd(source.parent) as source_fd,
310 _directory_fd(target.parent) as target_fd,
311 ):
312 if rename(
313 source_fd,
314 os.fsencode(source.name),
315 target_fd,
316 os.fsencode(target.name),
317 _RENAME_NOREPLACE,
318 ):
319 code = ctypes.get_errno()
320 if code == errno.EEXIST:
321 raise TrashError('restore_collision')
322 if code in {errno.EINVAL, errno.ENOSYS, errno.EOPNOTSUPP}:
323 # The filesystem (or kernel) can't promise not to
324 # replace: refuse rather than fall back to rename().
325 raise TrashError('atomic_move_unsupported')
326 raise OSError(code, os.strerror(code))
327 elif os.name == 'nt':
328 # Windows rename fails if the destination already exists.
329 os.rename(source, target)
330 else:
331 raise TrashError('atomic_move_unsupported')
332 _fsync_dir(source.parent)
333 _fsync_dir(target.parent)
336@contextmanager
337def _directory_fd(path: Path):
338 """Open each directory component without ever following a symlink."""
339 path = path.absolute()
340 fd = os.open(path.anchor, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
341 try:
342 for part in path.parts[1:]:
343 next_fd = os.open(
344 part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd
345 )
346 os.close(fd)
347 fd = next_fd
348 yield fd
349 finally:
350 os.close(fd)
353def _mkdir_beneath(root: Path, relative: str) -> None:
354 relative = _relative(relative, internal=True)
355 if os.name == 'nt':
356 _safe(root, relative, internal=True).mkdir(
357 parents=True, exist_ok=True, mode=0o700
358 )
359 return
360 with _directory_fd(root) as root_fd:
361 current = os.dup(root_fd)
362 try:
363 for part in PurePosixPath(relative).parts:
364 try:
365 os.mkdir(part, mode=0o700, dir_fd=current)
366 os.fsync(current)
367 except FileExistsError:
368 pass
369 next_fd = os.open(
370 part,
371 os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
372 dir_fd=current,
373 )
374 os.close(current)
375 current = next_fd
376 finally:
377 os.close(current)
380def _unlink_verified(path: Path, expected: dict) -> None:
381 actual = fingerprint(path)
382 if any(actual[k] != expected[k] for k in ('size', 'sha256')):
383 raise TrashError('trash_file_changed')
384 if os.name == 'nt':
385 path.unlink()
386 return
387 with _directory_fd(path.parent) as directory:
388 current = os.stat(path.name, dir_fd=directory, follow_symlinks=False)
389 if (
390 not stat.S_ISREG(current.st_mode)
391 or current.st_dev != actual['dev']
392 or current.st_ino != actual['ino']
393 or current.st_size != actual['size']
394 or current.st_mtime_ns != actual['mtime_ns']
395 ):
396 raise TrashError('trash_file_changed')
397 os.unlink(path.name, dir_fd=directory)
398 os.fsync(directory)
401@dataclass
402class _Hidden:
403 revision: int
404 files: frozenset[str]
405 resolved: frozenset[str]
406 checked: float
409class LibraryTrash:
410 def __init__(
411 self,
412 db: str | Path | Database,
413 roots: dict[str, TrashRoot],
414 *,
415 clock: Callable[[], float] = time.time,
416 worker: str = 'embedded',
417 ):
418 if worker not in WORKER_MODES:
419 raise TrashError('unsupported_configuration')
420 self.db = as_database(db, 'library')
421 # The file behind a SQLite database; None on PostgreSQL.
422 self._path: Optional[Path] = self.db.path
423 self.roots = roots
424 self.clock = clock
425 self.worker = worker
426 self._hidden: Optional[_Hidden] = None
427 self._init_db()
429 def _init_db(self) -> None:
430 # PostgreSQL gets its tables from the migrations; the revision seed
431 # row is needed on both backends.
432 if self.db.dialect is Dialect.SQLITE:
433 with self.connect() as conn:
434 conn.executescript("""
435 CREATE TABLE IF NOT EXISTS library_trash (
436 id TEXT PRIMARY KEY, batch TEXT NOT NULL,
437 user_id INTEGER NOT NULL, root_key TEXT NOT NULL,
438 relative TEXT NOT NULL, stored TEXT NOT NULL,
439 track_id TEXT NOT NULL DEFAULT '', row_json TEXT NOT NULL,
440 manifest TEXT NOT NULL, state TEXT NOT NULL,
441 action TEXT NOT NULL DEFAULT 'move', created REAL NOT NULL,
442 moved REAL, expires REAL, error TEXT NOT NULL DEFAULT ''
443 );
444 CREATE INDEX IF NOT EXISTS idx_trash_batch ON library_trash(batch);
445 CREATE INDEX IF NOT EXISTS idx_trash_stored ON library_trash(stored,state);
446 CREATE TABLE IF NOT EXISTS library_trash_blocks (
447 identity TEXT NOT NULL, item_id TEXT NOT NULL,
448 PRIMARY KEY(identity, item_id)
449 );
450 CREATE TABLE IF NOT EXISTS library_trash_meta (
451 key TEXT PRIMARY KEY, value INTEGER NOT NULL
452 );
453 """)
454 with self.db.transaction(lock=True) as conn:
455 conn.execute(
456 'INSERT INTO library_trash_meta (key,value) '
457 "VALUES ('revision',0) ON CONFLICT (key) DO NOTHING"
458 )
459 if self.db.dialect is Dialect.SQLITE:
460 self._migrate_sqlite(conn)
462 @staticmethod
463 def _migrate_sqlite(conn: Connection) -> None:
464 """Bring a development SQLite journal up to the current shape."""
465 block_columns = conn.execute(
466 'PRAGMA table_info(library_trash_blocks)'
467 ).fetchall()
468 if not any(row[1] == 'item_id' and row[5] for row in block_columns):
469 # Old development journals used only identity as the key;
470 # several independently trashed copies must remain excluded.
471 conn.execute(
472 'CREATE TABLE library_trash_blocks_v2 (identity TEXT NOT NULL, item_id TEXT NOT NULL, PRIMARY KEY(identity,item_id))'
473 )
474 conn.execute(
475 'INSERT OR IGNORE INTO library_trash_blocks_v2 '
476 'SELECT identity,item_id FROM library_trash_blocks'
477 )
478 conn.execute('DROP TABLE library_trash_blocks')
479 conn.execute(
480 'ALTER TABLE library_trash_blocks_v2 RENAME TO library_trash_blocks'
481 )
482 columns = {
483 r[1] for r in conn.execute('PRAGMA table_info(library_trash)')
484 }
485 if 'blocked' not in columns:
486 conn.execute(
487 'ALTER TABLE library_trash ADD COLUMN blocked INTEGER NOT NULL DEFAULT 1'
488 )
489 if 'requested_by' not in columns:
490 conn.execute(
491 'ALTER TABLE library_trash ADD COLUMN requested_by INTEGER NOT NULL DEFAULT 0'
492 )
494 def connect(self):
495 """A connection to the journal, re-entrant per thread."""
496 return self.db.connect()
498 def _row_lock(self) -> str:
499 """Lock the rows a read-then-write sequence depends on (PostgreSQL)."""
500 return ' FOR UPDATE' if self.db.dialect is Dialect.POSTGRES else ''
502 def _claim_lock(self) -> str:
503 """Pick work without waiting on rows another worker already holds."""
504 if self.db.dialect is Dialect.POSTGRES:
505 return ' FOR UPDATE SKIP LOCKED'
506 return ''
508 def _root(self, key: str) -> Path:
509 if key not in self.roots:
510 raise TrashError('unsupported_collection')
511 return self.roots[key].check()
513 def preview(self, user_id: int, tracks: list[dict]) -> dict:
514 if user_id <= 0 or not tracks or len(tracks) > 200:
515 raise TrashError('invalid_selection')
516 batch = secrets.token_urlsafe(32)
517 rows = []
518 seen = set()
519 for track in tracks:
520 stored = str(track['stored'])
521 if stored in seen:
522 continue
523 seen.add(stored)
524 root = self._root(track['root_key'])
525 relative = _relative(track['relative'])
526 audio = _safe(root, relative)
527 item_id = secrets.token_hex(16)
528 if not identity_keys(track.get('metadata', {})):
529 raise TrashError('identity_uncertain')
530 manifest = []
531 sources = [(track['root_key'], root, audio)]
532 shared_sidecar = any(
533 p != audio
534 and p.suffix.lower()
535 in {'.mp3', '.flac', '.m4a', '.ogg', '.opus', '.wav', '.aac'}
536 and p.stem == audio.stem
537 for p in audio.parent.iterdir()
538 )
539 # Only track-specific sidecars; never shared folder artwork.
540 for suffix in ('.lrc', '.txt', '.srt'):
541 sidecar = audio.with_suffix(suffix)
542 if sidecar.exists() or sidecar.is_symlink():
543 if shared_sidecar:
544 continue
545 sources.append((track['root_key'], root, sidecar))
546 if not shared_sidecar:
547 for sidecar in track.get('extra_sidecars', []):
548 side_root = self._root(sidecar['root_key'])
549 side_rel = _relative(sidecar['relative'])
550 source = _safe(side_root, side_rel)
551 if source.suffix.lower() != '.lrc':
552 raise TrashError('invalid_path')
553 if source.exists() and not any(
554 source == item[2] for item in sources
555 ):
556 sources.append((
557 sidecar['root_key'],
558 side_root,
559 source,
560 ))
561 for index, (source_key, source_root, source) in enumerate(sources):
562 rel = source.relative_to(source_root).as_posix()
563 _safe(source_root, rel)
564 manifest.append({
565 'source': rel,
566 'target': f'{TRASH_DIR}/{item_id}/{index}{source.suffix.lower()}',
567 'root_key': source_key,
568 'fingerprint': fingerprint(source),
569 })
570 rows.append({
571 'id': item_id,
572 'batch': batch,
573 'user_id': user_id,
574 'root_key': track['root_key'],
575 'relative': relative,
576 'stored': stored,
577 'track_id': track.get('track_id', ''),
578 'row_json': json.dumps(track.get('metadata', {})),
579 'manifest': json.dumps(manifest),
580 'state': 'preview',
581 'created': self.clock(),
582 })
583 with self.connect() as conn:
584 for row in rows:
585 conn.execute(
586 """INSERT INTO library_trash
587 (id,batch,user_id,root_key,relative,stored,track_id,row_json,manifest,state,created)
588 VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
589 tuple(row[k] for k in _TRASH_COLUMNS),
590 )
591 return {
592 'confirmation': batch,
593 'expires_at': self.clock() + PREVIEW_SECONDS,
594 'items': [self._public(row) for row in rows],
595 }
597 @staticmethod
598 def _public(row) -> dict:
599 row = dict(row)
600 return {
601 'id': row['id'],
602 'file': row['stored'],
603 'track_id': row.get('track_id', ''),
604 'metadata': json.loads(row['row_json']),
605 'state': row['state'],
606 'size': sum(
607 x['fingerprint']['size'] for x in json.loads(row['manifest'])
608 ),
609 'expires_at': row.get('expires'),
610 'error': row.get('error', ''),
611 }
613 def confirm(self, user_id: int, batch: str) -> list[str]:
614 with self.db.transaction(lock=True) as conn:
615 rows = conn.execute(
616 'SELECT * FROM library_trash WHERE batch=? AND user_id=?'
617 + self._row_lock(),
618 (batch, user_id),
619 ).fetchall()
620 if not rows:
621 raise TrashError('confirmation_invalid')
622 if all(r['state'] != 'preview' for r in rows):
623 return [r['id'] for r in rows]
624 if any(
625 r['state'] != 'preview'
626 or self.clock() - r['created'] > PREVIEW_SECONDS
627 for r in rows
628 ):
629 raise TrashError('confirmation_expired')
630 for row in rows:
631 if conn.execute(
632 "SELECT 1 FROM library_trash WHERE stored=? AND state NOT IN ('preview','restored','purged')",
633 (row['stored'],),
634 ).fetchone():
635 raise TrashError('operation_already_pending')
636 for file in json.loads(row['manifest']):
637 root = self._root(file.get('root_key', row['root_key']))
638 if not _matches(
639 _safe(root, file['source']),
640 file['fingerprint'],
641 original=True,
642 ):
643 raise TrashError('file_changed')
644 conn.execute(
645 "UPDATE library_trash SET state='pending' WHERE batch=?",
646 (batch,),
647 )
648 conn.execute(
649 "UPDATE library_trash_meta SET value=value+1 WHERE key='revision'"
650 )
651 for row in rows:
652 for key in identity_keys(json.loads(row['row_json'])):
653 conn.execute(
654 'INSERT INTO library_trash_blocks(identity,item_id) VALUES (?,?) '
655 'ON CONFLICT (identity,item_id) DO NOTHING',
656 (key, row['id']),
657 )
658 self._hidden = None
659 return [r['id'] for r in rows]
661 def confirmation_files(self, user_id: int, batch: str) -> list[str]:
662 with self.connect() as conn:
663 return [
664 r[0]
665 for r in conn.execute(
666 'SELECT stored FROM library_trash WHERE batch=? AND user_id=?',
667 (batch, user_id),
668 )
669 ]
671 def list(
672 self, *, offset: int = 0, limit: int = 50, search: str = ''
673 ) -> dict:
674 with self.connect() as conn:
675 rows = conn.execute(
676 "SELECT * FROM library_trash WHERE state NOT IN ('preview','restored','purged') ORDER BY created DESC"
677 ).fetchall()
678 matching = [
679 r
680 for r in rows
681 if search.casefold() in (r['stored'] + r['row_json']).casefold()
682 ]
683 return {
684 'items': [
685 self._public(r)
686 for r in matching[offset : offset + min(limit, 100)]
687 ],
688 'total': len(matching),
689 'size': sum(self._public(r)['size'] for r in matching),
690 }
692 def request_restore(self, ids: list[str], *, user_id: int = 0) -> None:
693 if not ids or len(ids) > 200:
694 raise TrashError('invalid_selection')
695 with self.db.transaction(lock=True) as conn:
696 for item in set(ids):
697 row = conn.execute(
698 'SELECT * FROM library_trash WHERE id=?'
699 + self._row_lock(),
700 (item,),
701 ).fetchone()
702 if row is None or row['state'] == 'preview':
703 raise TrashError('item_not_found')
704 if row['state'] == 'restored':
705 continue
706 if row['state'] in {'moving', 'restoring'}:
707 raise TrashError('operation_in_progress')
708 if row['state'] in {'purging', 'purged'} or (
709 row['expires'] is not None
710 and self.clock() >= row['expires']
711 ):
712 raise TrashError('retention_expired')
713 conn.execute(
714 "UPDATE library_trash SET state='restore_pending', action='restore', requested_by=?, error='' WHERE id=?",
715 (user_id or row['user_id'], item),
716 )
718 def hidden(self) -> tuple[frozenset[str], frozenset[str]]:
719 """The library paths kept out of every listing, as stored and as
720 resolved on disk. Read on every request, so cached until the
721 journal's revision moves."""
723 now = time.monotonic()
724 cached = self._hidden
725 if cached is not None and now - cached.checked < (
726 HIDDEN_RECHECK_SECONDS
727 ):
728 return cached.files, cached.resolved
729 with self.connect() as conn:
730 revision = conn.execute(
731 "SELECT value FROM library_trash_meta WHERE key='revision'"
732 ).fetchone()[0]
733 if cached is not None and cached.revision == revision:
734 cached.checked = now
735 return cached.files, cached.resolved
736 rows = conn.execute(
737 "SELECT stored,root_key,relative FROM library_trash WHERE state NOT IN ('preview','restored') AND blocked=1"
738 ).fetchall()
739 bases = {}
740 for key, root in self.roots.items():
741 try:
742 bases[key] = root.path.absolute().resolve()
743 except OSError:
744 continue
745 files = frozenset(r['stored'] for r in rows)
746 resolved = frozenset(
747 str(bases[r['root_key']] / r['relative'])
748 for r in rows
749 if r['root_key'] in bases
750 )
751 self._hidden = _Hidden(revision, files, resolved, now)
752 return files, resolved
754 def hidden_paths(self) -> frozenset[str]:
755 return self.hidden()[0]
757 def hidden_resolved(self) -> frozenset[str]:
758 return self.hidden()[1]
760 def _update(self, item: str, **values) -> None:
761 with self.connect() as conn:
762 conn.execute(
763 'UPDATE library_trash SET '
764 + ','.join(k + '=?' for k in values)
765 + ' WHERE id=?',
766 (*values.values(), item),
767 )
768 if 'blocked' in values or values.get('state') == 'restored':
769 conn.execute(
770 "UPDATE library_trash_meta SET value=value+1 WHERE key='revision'"
771 )
772 self._hidden = None
774 def revision(self) -> int:
775 with self.connect() as conn:
776 return conn.execute(
777 "SELECT value FROM library_trash_meta WHERE key='revision'"
778 ).fetchone()[0]
780 def process(
781 self,
782 *,
783 limit: int = 10,
784 authorized: Callable[[int], bool],
785 skip_ids: frozenset[str] = frozenset(),
786 ) -> list[dict]:
787 if self._path is None:
788 # PostgreSQL has no file to lock. Every row is claimed by an
789 # atomic UPDATE in _process_item, so two workers never move one.
790 return self._process(
791 limit=limit, authorized=authorized, skip_ids=skip_ids
792 )
793 with worker_lock(
794 self._path.with_suffix('.trash-worker.lock')
795 ) as acquired:
796 if not acquired:
797 return []
798 return self._process(
799 limit=limit, authorized=authorized, skip_ids=skip_ids
800 )
802 def blocked(self, song: dict) -> bool:
803 with self.connect() as conn:
804 for key in identity_keys(song):
805 if conn.execute(
806 """SELECT 1 FROM library_trash_blocks b JOIN library_trash t ON t.id=b.item_id
807 WHERE b.identity=? AND t.blocked=1 AND t.state NOT IN ('preview','restored')""",
808 (key,),
809 ).fetchone():
810 return True
811 return False
813 def allow_download(self, ids: list[str]) -> None:
814 if not ids or len(ids) > 200:
815 raise TrashError('invalid_selection')
816 with self.connect() as conn:
817 for item in ids:
818 row = conn.execute(
819 'SELECT state FROM library_trash WHERE id=?'
820 + self._row_lock(),
821 (item,),
822 ).fetchone()
823 if row is None or row['state'] not in {
824 'trashed',
825 'purged',
826 'restored',
827 }:
828 raise TrashError('operation_not_complete')
829 conn.execute(
830 'UPDATE library_trash SET blocked=0 WHERE id=?', (item,)
831 )
832 conn.execute(
833 "UPDATE library_trash_meta SET value=value+1 WHERE key='revision'"
834 )
835 self._hidden = None
837 def retry(self, ids: list[str], *, user_id: int) -> None:
838 if not ids or len(ids) > 200:
839 raise TrashError('invalid_selection')
840 with self.connect() as conn:
841 for item in ids:
842 row = conn.execute(
843 'SELECT * FROM library_trash WHERE id=?'
844 + self._row_lock(),
845 (item,),
846 ).fetchone()
847 if row is None or row['state'] != 'error':
848 raise TrashError('operation_not_complete')
849 state = {
850 'move': 'pending',
851 'restore': 'restore_pending',
852 'purge': 'purging',
853 }[row['action']]
854 conn.execute(
855 'UPDATE library_trash SET state=?,requested_by=?,error=? WHERE id=?',
856 (state, user_id, '', item),
857 )
859 def has_work(self) -> bool:
860 """Whether a worker pass would do anything (cheap: one query)."""
862 with self.connect() as conn:
863 return (
864 conn.execute(
865 'SELECT 1 ' + _WORK + ' LIMIT 1',
866 (self.clock(),),
867 ).fetchone()
868 is not None
869 )
871 def work_files(self) -> dict[str, str]:
872 """``{operation id: library path}`` of the work waiting, so the
873 embedded worker can hold back a song that is playing."""
875 with self.connect() as conn:
876 return {
877 r['id']: r['stored']
878 for r in conn.execute(
879 'SELECT id,stored ' + _WORK, (self.clock(),)
880 )
881 }
883 def exclusions(self) -> list[dict]:
884 with self.connect() as conn:
885 return [
886 self._public(r)
887 for r in conn.execute(
888 "SELECT * FROM library_trash WHERE blocked=1 AND state='purged' ORDER BY created DESC"
889 )
890 ]
892 def _process(
893 self,
894 *,
895 limit: int,
896 authorized: Callable[[int], bool],
897 skip_ids: frozenset[str],
898 ) -> list[dict]:
899 """One bounded, single-worker pass. Authorization is rechecked here."""
900 result = []
901 with self.connect() as conn:
902 rows = conn.execute(
903 'SELECT * '
904 + _WORK
905 + ' ORDER BY created LIMIT ?'
906 + self._claim_lock(),
907 (self.clock(), limit + len(skip_ids)),
908 ).fetchall()
909 for row in rows:
910 if row['id'] in skip_ids:
911 continue
912 if len(result) >= limit:
913 break
914 try:
915 if self._process_item(dict(row), authorized=authorized):
916 result.append({'id': row['id'], 'ok': True})
917 except (OSError, TrashError) as exc:
918 error = (
919 str(exc)
920 if isinstance(exc, TrashError)
921 else 'filesystem_unavailable'
922 )
923 self._update(row['id'], state='error', error=error)
924 result.append({'id': row['id'], 'ok': False, 'error': error})
925 return result
927 def _process_item(self, row: dict, *, authorized=None) -> bool:
928 action = row['action']
929 if row['state'] in {'trashed', 'purging'}:
930 if row['expires'] is None or self.clock() < row['expires']:
931 raise TrashError('retention_not_expired')
932 action = 'purge'
933 next_state = {
934 'move': 'moving',
935 'restore': 'restoring',
936 'purge': 'purging',
937 }[action]
938 with self.connect() as conn:
939 claimed = conn.execute(
940 'UPDATE library_trash SET state=?,action=? WHERE id=? AND state=? AND action=?',
941 (next_state, action, row['id'], row['state'], row['action']),
942 ).rowcount
943 if not claimed:
944 return False
945 if action in {'move', 'restore'} and authorized is not None:
946 if not authorized(row['requested_by'] or row['user_id']):
947 raise TrashError('admin_access_revoked')
948 files = json.loads(row['manifest'])
949 roots = {}
950 for file in files:
951 key = file.get('root_key', row['root_key'])
952 if key not in roots:
953 roots[key] = self._root(key)
954 if action == 'move':
955 for root in roots.values():
956 trash = _safe(root, TRASH_DIR, internal=True)
957 _mkdir_beneath(root, TRASH_DIR)
958 if os.name != 'nt' and trash.stat().st_mode & 0o077:
959 raise TrashError('trash_permissions_unsafe')
960 ignore = trash / '.ignore'
961 if ignore.is_symlink():
962 raise TrashError('unsafe_symlink')
963 if not ignore.exists():
964 with ignore.open('xb'):
965 pass
966 _fsync_dir(trash)
967 if ignore.stat().st_size:
968 raise TrashError('trash_ignore_invalid')
969 _safe(root, f'{TRASH_DIR}/{row["id"]}', internal=True)
970 _mkdir_beneath(root, f'{TRASH_DIR}/{row["id"]}')
971 # Validate the whole operation before the first move/unlink.
972 for file in files:
973 root = roots[file.get('root_key', row['root_key'])]
974 source = _safe(root, file['source'])
975 target = _safe(root, file['target'], internal=True)
976 current = source if action == 'move' else target
977 other = target if action == 'move' else source
978 if action == 'purge':
979 if target.exists():
980 if not _matches(target, file['fingerprint']):
981 raise TrashError('trash_file_changed')
982 elif not file.get('purge_intent'):
983 raise TrashError('trash_file_missing')
984 elif current.exists():
985 if other.exists():
986 raise TrashError('restore_collision')
987 if not _matches(
988 current, file['fingerprint'], original=action == 'move'
989 ):
990 raise TrashError('file_changed')
991 elif action == 'restore' and other.exists():
992 # Never moved (a move that failed or was interrupted
993 # first): the file is still where it was, whatever has
994 # happened to it since. Restoring only shows it again.
995 continue
996 elif not other.exists() or not _matches(
997 other, file['fingerprint']
998 ):
999 raise TrashError('file_missing')
1000 for file in files:
1001 root = roots[file.get('root_key', row['root_key'])]
1002 source = _safe(root, file['source'])
1003 target = _safe(root, file['target'], internal=True)
1004 if action == 'purge':
1005 if target.exists():
1006 if not _matches(target, file['fingerprint']):
1007 raise TrashError('trash_file_changed')
1008 file['purge_intent'] = True
1009 self._update(row['id'], manifest=json.dumps(files))
1010 _unlink_verified(target, file['fingerprint'])
1011 elif action == 'move' and source.exists():
1012 if not _matches(source, file['fingerprint'], original=True):
1013 raise TrashError('file_changed')
1014 _rename_no_replace(source, target)
1015 if not _matches(target, file['fingerprint']):
1016 raise TrashError('file_changed_during_move')
1017 elif action == 'restore' and target.exists():
1018 parent = source.parent.relative_to(root).as_posix()
1019 if parent != '.':
1020 _mkdir_beneath(root, parent)
1021 _safe(root, file['source'])
1022 if not _matches(target, file['fingerprint']):
1023 raise TrashError('trash_file_changed')
1024 _rename_no_replace(target, source)
1025 if action == 'move':
1026 moved = self.clock()
1027 self._update(
1028 row['id'],
1029 state='trashed',
1030 moved=moved,
1031 expires=moved + RETENTION_SECONDS,
1032 error='',
1033 )
1034 elif action == 'restore':
1035 self._update(row['id'], state='restored', blocked=0, error='')
1036 else:
1037 self._update(row['id'], state='purged', error='')
1038 return True
1041def trash_from_config(
1042 config_path: Path, db: str | Path | Database
1043) -> LibraryTrash:
1044 config = json.loads(config_path.read_text(encoding='utf-8'))
1045 if config.get('version') != 1:
1046 raise TrashError('unsupported_configuration')
1047 roots = {}
1048 for key, value in config['roots'].items():
1049 path = Path(value['path'])
1050 if not path.is_absolute():
1051 raise TrashError('invalid_root')
1052 marker = str(value.get('marker', ''))
1053 if marker:
1054 _relative(marker)
1055 roots[key] = TrashRoot(
1056 path, marker, str(value.get('marker_value', ''))
1057 )
1058 if not roots:
1059 raise TrashError('invalid_root')
1060 return LibraryTrash(
1061 db, roots, worker=str(config.get('worker', 'embedded'))
1062 )
1065def trash_from_env(
1066 download_dir: Path,
1067 db: str | Path | Database,
1068 env: Optional[Mapping[str, str]] = None,
1069) -> Optional[LibraryTrash]:
1070 """The trash the operator turned on, or ``None``.
1072 ``DOWNTIFY_TRASH_CONFIG`` (a JSON file naming each folder, and where
1073 the worker runs) wins; otherwise ``DOWNTIFY_TRASH=true`` keeps it to
1074 the download folder, with the worker inside the app."""
1076 env = os.environ if env is None else env
1077 config = str(env.get('DOWNTIFY_TRASH_CONFIG') or '').strip()
1078 if config:
1079 return trash_from_config(Path(config), db)
1080 if str(env.get('DOWNTIFY_TRASH') or '').strip().lower() in _TRUE:
1081 return LibraryTrash(
1082 db, {'downloads': TrashRoot(Path(download_dir).absolute())}
1083 )
1084 return None