diff --git a/TODO.md b/TODO.md index 7d8c20b..7a49b5c 100755 --- a/TODO.md +++ b/TODO.md @@ -14,6 +14,7 @@ update m3u8 matching files in spotify to files in collection ### prepare for decks +store ANLZ in metadata directory by isrc create data/usb directory convert to mp3 v0 jstereo - output into the data/usb directory diff --git a/justfile b/justfile index b171d9b..bef4f16 100644 --- a/justfile +++ b/justfile @@ -40,13 +40,50 @@ sockseek input: collection ids="7f5tPIfj21UJI4zfDmH4rx 6sAl8LhU6ediFNJRuQQIQX": cd maple-py && uv run python -m maple.collection {{ids}} +# Build a rekordbox USB layout at from the collection at . +# Converts FLAC→MP3 V0, extracts artwork, writes ANLZ (with PWAV/PWV3 +# blue/white waveforms) and PIONEER/rekordbox/export.pdb. +export src="data/collection" dst="data/usb": + cd maple-py && uv run python -m maple "{{justfile_directory() / src}}" "{{justfile_directory() / dst}}" + sync args: s5cmd {{args}} -usb dst="/Volumes/MAPLE": - COPYFILE_DISABLE=1 rsync -rtvh --delete --size-only --no-perms --no-owner --no-group --modify-window=2 maple-py/data/usb/Contents/ {{dst}}/Contents/ - COPYFILE_DISABLE=1 rsync -rtvh --delete --size-only --no-perms --no-owner --no-group --modify-window=2 --exclude='USBANLZ/' maple-py/data/usb/PIONEER/ {{dst}}/PIONEER/ - rm -f {{dst}}/PIONEER/rekordbox/RBFLTR.DAT +# Sync data/usb/ → mounted USB stick (default /media/ntr/MAPLE), then +# safely unmount so the stick can be pulled without risk to the FAT +# write-back cache. +usb dst="/media/ntr/MAPLE": + @test -d {{dst}} || (echo "USB not mounted at {{dst}}"; exit 1) + rsync -rtvh --delete --size-only --no-perms --no-owner --no-group --modify-window=2 "{{justfile_directory()}}/data/usb/" "{{dst}}/" + sync + @# Try udisksctl first (no sudo, handles the udev event); fall back to + @# plain `umount` if udisks isn't available. Either way the recipe + @# exits 0 even if the unmount fails (e.g. someone has a file open). + @udisksctl unmount -b "$(findmnt -no SOURCE {{dst}})" 2>/dev/null && echo "unmounted {{dst}}" \ + || (umount {{dst}} 2>/dev/null && echo "unmounted {{dst}}") \ + || echo "warning: could not unmount {{dst}} — close any file browsers + retry" + +# Build a FAT32 image of data/usb/ — ready to dd onto a stick. +# `size` must exceed the data/usb/ tree (rough rule: data size + ~50 MB FAT overhead). +usb-image size="2G" img="data/maple.img": + @command -v mcopy >/dev/null || (echo "mtools not installed (apt install mtools)"; exit 1) + @test -d "{{justfile_directory()}}/data/usb" || (echo "no data/usb (run \`just export\` first)"; exit 1) + rm -f "{{justfile_directory()}}/{{img}}" + truncate -s {{size}} "{{justfile_directory()}}/{{img}}" + mkfs.vfat -F 32 -n MAPLE "{{justfile_directory()}}/{{img}}" + cd "{{justfile_directory()}}/data/usb" && mcopy -i "{{justfile_directory()}}/{{img}}" -s -Q -p -m * ::/ + @echo "image ready:" && ls -lh "{{justfile_directory()}}/{{img}}" + +# DESTRUCTIVE: dd data/maple.img onto a block device (e.g. /dev/sdb). +# Double-check `lsblk` first — wrong device wipes that drive. +[confirm("usb-flash will OVERWRITE the entire device. Proceed?")] +usb-flash dev img="data/maple.img": + @test -b {{dev}} || (echo "{{dev}} is not a block device"; exit 1) + @test -f "{{justfile_directory()}}/{{img}}" || (echo "no image at {{img}}; run \`just usb-image\` first"; exit 1) + @lsblk -no NAME,SIZE,LABEL,MODEL {{dev}} + sudo dd if="{{justfile_directory()}}/{{img}}" of={{dev}} bs=4M status=progress oflag=direct conv=fsync + sudo sync + @echo "done — safe to remove (try: udisksctl power-off -b {{dev}})" usb-pdb dst="/Volumes/MAPLE": @test -d {{dst}}/PIONEER/rekordbox || (echo "no rekordbox dir at {{dst}}/PIONEER"; exit 1) diff --git a/maple-py/pyproject.toml b/maple-py/pyproject.toml index e178363..82db32d 100644 --- a/maple-py/pyproject.toml +++ b/maple-py/pyproject.toml @@ -9,10 +9,12 @@ authors = [ requires-python = ">=3.14" dependencies = [ "aiohttp>=3.10", + "aubio>=0.4.9", "kaitaistruct>=0.11", "loguru>=0.7", "mutagen>=1.47.0", "pillow>=12.2.0", + "polars>=1.0", "pyrekordbox>=0.4.4", ] diff --git a/maple-py/pyrightconfig.json b/maple-py/pyrightconfig.json new file mode 100644 index 0000000..8a8a426 --- /dev/null +++ b/maple-py/pyrightconfig.json @@ -0,0 +1,23 @@ +{ + "include": [ + "src", + "tests" + ], + "extraPaths": [ + "src" + ], + "exclude": [ + "**/__pycache__", + "**/.venv", + "data", + "re" + ], + "venvPath": ".", + "venv": ".venv", + "pythonVersion": "3.14", + "pythonPlatform": "Linux", + "typeCheckingMode": "basic", + "reportMissingImports": "error", + "reportMissingModuleSource": "warning", + "useLibraryCodeForTypes": true +} diff --git a/maple-py/src/maple/analyze.py b/maple-py/src/maple/analyze.py new file mode 100644 index 0000000..301b255 --- /dev/null +++ b/maple-py/src/maple/analyze.py @@ -0,0 +1,293 @@ +"""Audio analysis: analytical BPM detection + waveform envelope extraction. + +ffmpeg decodes the source to 44.1 kHz mono float32 PCM on stdout. The +samples are then either streamed in hop-size chunks into aubio.tempo for +BPM detection, or fully buffered and binned to produce the per-column +amplitude envelopes that PWAV / PWV3 ANLZ sections embed. + +The decoded audio never touches disk — ffmpeg's `-f f32le` output is +piped straight to numpy. +""" +from __future__ import annotations + +import json +import shutil +import subprocess +from base64 import b64decode, b64encode +from pathlib import Path + +import aubio +import numpy as np +import polars as pl +from loguru import logger + +HOP_SIZE = 512 +WIN_SIZE = 1024 +SAMPLE_RATE = 44100 +MIN_BEATS = 8 + +# Waveform envelope dimensions. +PREVIEW_COLS = 400 # PWAV is fixed at 400 columns (1 byte/col). +SCROLL_FPS = 150 # PWV3 = ~150 columns/sec across the whole track. +PWAV_LEVELS = 31 # PWAV/PWV3 stores 5-bit heights (bit 5 = white flag). + + +def _stream_pcm(path: Path): + """Yield successive hop-sized float32 numpy arrays of mono PCM. + + Pads the final chunk with zeros so every yield is exactly HOP_SIZE samples + (aubio.tempo requires fixed-size inputs). + """ + proc = subprocess.Popen( + ["ffmpeg", "-nostdin", "-v", "error", + "-i", str(path), + "-f", "f32le", "-ac", "1", "-ar", str(SAMPLE_RATE), "-"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + assert proc.stdout is not None + bytes_per_chunk = HOP_SIZE * 4 # float32 + while True: + buf = proc.stdout.read(bytes_per_chunk) + if not buf: + break + arr = np.frombuffer(buf, dtype=np.float32) + if len(arr) < HOP_SIZE: + pad = np.zeros(HOP_SIZE - len(arr), dtype=np.float32) + arr = np.concatenate([arr, pad]) + yield arr + finally: + if proc.poll() is None: + proc.terminate() + proc.wait() + + +def detect_bpm(path: Path) -> float | None: + """Return the analytical BPM of `path`, or None on failure. + + Uses median-of-intervals (via polars), which is robust against + aubio's "live" running-average swings on transient sections. + """ + if not shutil.which("ffmpeg"): + logger.warning("ffmpeg not on PATH; BPM detection disabled") + return None + + tempo = aubio.tempo("default", WIN_SIZE, HOP_SIZE, SAMPLE_RATE) + beats: list[float] = [] + try: + for chunk in _stream_pcm(path): + if tempo(chunk): + beats.append(tempo.get_last_s()) + except Exception as e: + logger.warning("BPM detection failed for {}: {}", path.name, e) + return None + + if len(beats) < MIN_BEATS: + return None + intervals = pl.Series( + [beats[i + 1] - beats[i] for i in range(len(beats) - 1)] + ) + median_interval = intervals.median() + if not median_interval or median_interval <= 0: + return None + return 60.0 / median_interval + + +def _decode_pcm(path: Path) -> np.ndarray: + """Decode `path` to a 1D float32 numpy array of mono PCM at SAMPLE_RATE. + + Returns an empty array on ffmpeg failure. + """ + if not shutil.which("ffmpeg"): + logger.warning("ffmpeg not on PATH; cannot extract waveform") + return np.empty(0, dtype=np.float32) + proc = subprocess.run( + ["ffmpeg", "-nostdin", "-v", "error", + "-i", str(path), + "-f", "f32le", "-ac", "1", "-ar", str(SAMPLE_RATE), "-"], + capture_output=True, + ) + if proc.returncode != 0: + logger.warning("ffmpeg failed for {}: {}", path.name, + proc.stderr.decode("utf-8", "replace").strip()) + return np.empty(0, dtype=np.float32) + return np.frombuffer(proc.stdout, dtype=np.float32) + + +def _bin_amplitudes(samples: np.ndarray, n_bins: int, levels: int = PWAV_LEVELS) -> bytes: + """Split `samples` into `n_bins` contiguous windows, take the peak |s| per + window, quantise to 0..levels, and return one byte per bin. + + Quantising at 5-bit resolution matches PWAV's `bits 0-4 = height` encoding; + bit 5 (the "white overshoot" flag) is left at 0 for now — most CDJ firmware + is happy without it. + """ + if n_bins <= 0 or len(samples) == 0: + return bytes(n_bins) + chunks = np.array_split(np.abs(samples), n_bins) + peaks = np.array([float(c.max()) if len(c) else 0.0 for c in chunks]) + peak_max = float(peaks.max()) if peaks.size else 0.0 + if peak_max <= 0: + return bytes(n_bins) + q = np.clip(np.round(peaks / peak_max * levels), 0, levels).astype(np.uint8) + return q.tobytes() + + +def extract_waveform_envelope(path: Path) -> dict[str, bytes]: + """Return the two byte streams a PWAV + PWV3 pair needs. + + Output: + { + "preview": bytes of length PREVIEW_COLS (always 400), + "scroll": bytes of length ~ duration * SCROLL_FPS, + } + On decode failure both streams are empty. + """ + samples = _decode_pcm(path) + if samples.size == 0: + return {"preview": b"", "scroll": b""} + duration = samples.size / SAMPLE_RATE + scroll_cols = max(1, int(round(duration * SCROLL_FPS))) + return { + "preview": _bin_amplitudes(samples, PREVIEW_COLS), + "scroll": _bin_amplitudes(samples, scroll_cols), + } + + +def _detect_beats_in_samples(samples: np.ndarray) -> list[float]: + """Run aubio's onset detector across a pre-loaded mono PCM buffer. + + Returns the timestamps (seconds) of every detected beat. + """ + tempo = aubio.tempo("default", WIN_SIZE, HOP_SIZE, SAMPLE_RATE) + beats: list[float] = [] + for i in range(0, len(samples), HOP_SIZE): + chunk = samples[i:i + HOP_SIZE] + if len(chunk) < HOP_SIZE: + chunk = np.concatenate( + [chunk, np.zeros(HOP_SIZE - len(chunk), dtype=np.float32)] + ) + if tempo(chunk): + beats.append(tempo.get_last_s()) + return beats + + +def _project_beat_grid(beats: list[float], bpm: float, duration_s: float) -> list[tuple[int, int, int]]: + """Project a rigid `bpm` beat grid using the first detected beat as anchor. + + Returns `[(beat_number 1-4, tempo*100, time_ms), ...]` covering t=0 to + `duration_s`. Rigid grids are what CDJs prefer over noisy per-beat + timing — the strongest detected beat fixes the phase and we project + forward + backward at the canonical tempo. + """ + if not beats or bpm <= 0: + return [] + interval = 60.0 / bpm + first = beats[0] + # Walk back from the first detected beat to t≥0 so the grid covers the intro. + n_before = int(first / interval) + anchor = first - n_before * interval + if anchor < 0: + anchor += interval + n_before -= 1 + tempo_x100 = round(bpm * 100) + grid: list[tuple[int, int, int]] = [] + t = anchor + i = 0 + while t <= duration_s + 1.0: + time_ms = int(round(t * 1000)) + beat_number = (i % 4) + 1 + grid.append((beat_number, tempo_x100, time_ms)) + i += 1 + t = anchor + i * interval + return grid + + +CACHE_VERSION = 1 + + +def _sanitise_isrc(isrc: str) -> str: + """Reduce an ISRC to a filename-safe form (uppercase, alnum only).""" + return "".join(c for c in (isrc or "").upper() if c.isalnum()) + + +def cache_path_for(cache_dir: Path, isrc: str) -> Path | None: + """Return the path the cache file for `isrc` lives at, or None if no isrc.""" + s = _sanitise_isrc(isrc) + if not s: + return None + return cache_dir / f"{s}.json" + + +def save_analysis(cache_dir: Path, isrc: str, analysis: dict) -> Path | None: + """Persist `analysis` for `isrc` under `cache_dir`. No-op if isrc empty.""" + target = cache_path_for(cache_dir, isrc) + if target is None: + return None + target.parent.mkdir(parents=True, exist_ok=True) + payload = { + "version": CACHE_VERSION, + "isrc": _sanitise_isrc(isrc), + "bpm": analysis.get("bpm"), + "beats": analysis.get("beats", []), + "preview_b64": b64encode(analysis.get("preview") or b"").decode("ascii"), + "scroll_b64": b64encode(analysis.get("scroll") or b"").decode("ascii"), + } + target.write_text(json.dumps(payload, separators=(",", ":"))) + return target + + +def load_analysis(cache_dir: Path, isrc: str) -> dict | None: + """Return cached analysis for `isrc`, or None if absent/stale/unreadable.""" + target = cache_path_for(cache_dir, isrc) + if target is None or not target.exists(): + return None + try: + d = json.loads(target.read_text()) + if d.get("version") != CACHE_VERSION: + return None + return { + "bpm": d.get("bpm"), + # JSON tuples come back as lists; PQTZ writer accepts either. + "beats": [tuple(b) for b in d.get("beats", [])], + "preview": b64decode(d.get("preview_b64") or ""), + "scroll": b64decode(d.get("scroll_b64") or ""), + } + except (json.JSONDecodeError, OSError, ValueError) as e: + logger.warning("analysis cache read failed for {}: {}", isrc, e) + return None + + +def analyze_audio(path: Path) -> dict: + """One-pass decode + analysis. Returns: + { + "bpm": float | None, # median-of-intervals tempo + "beats": list[(beat_n, tempo_x100, time_ms)], # for PQTZ + "preview": bytes, # for PWAV (400-byte amplitude strip) + "scroll": bytes, # for PWV3 (~150 cols/sec) + } + """ + samples = _decode_pcm(path) + out: dict = {"bpm": None, "beats": [], "preview": b"", "scroll": b""} + if samples.size == 0: + return out + + # Beat onsets. + beat_times = _detect_beats_in_samples(samples) + if len(beat_times) >= MIN_BEATS: + intervals = pl.Series( + [beat_times[i + 1] - beat_times[i] for i in range(len(beat_times) - 1)] + ) + median_interval = intervals.median() + if median_interval and median_interval > 0: + out["bpm"] = 60.0 / median_interval + duration_s = samples.size / SAMPLE_RATE + out["beats"] = _project_beat_grid(beat_times, out["bpm"], duration_s) + + # Waveform envelopes. + duration = samples.size / SAMPLE_RATE + scroll_cols = max(1, int(round(duration * SCROLL_FPS))) + out["preview"] = _bin_amplitudes(samples, PREVIEW_COLS) + out["scroll"] = _bin_amplitudes(samples, scroll_cols) + return out diff --git a/maple-py/src/maple/anlz.py b/maple-py/src/maple/anlz.py index 87bf47a..a141d3a 100644 --- a/maple-py/src/maple/anlz.py +++ b/maple-py/src/maple/anlz.py @@ -1,41 +1,260 @@ -"""Generate a minimal rekordbox ANLZ0000.DAT analysis file. +"""Generate a rekordbox ANLZ0000.{DAT,EXT} pair. -The CDJ/XDJ reads the path from a `PPTH` section to confirm which audio -file the analysis applies to. We emit just that section -- no beat grid, -no waveforms -- which is enough for the player to consider the track's -analysis "present" and surface the track in the library. Without an -analyze file at the path the pdb references, some firmware hides the -track entirely. +ANLZ0000.DAT — what CDJ-900-era through CDJ-3000 firmware reads to display +the loaded-deck BPM, the touch-strip preview waveform, and the cue list: + + PPTH player-relative audio path + PVBR variable-bitrate seek index (zero-filled placeholder; matches the + shape rekordbox writes for files where the seek index isn't useful) + PQTZ beat grid (per-beat: beat_number, tempo*100, time_ms) + PWAV 400-byte blue/white touch-strip preview + PWV2 100-byte "tiny" overview waveform + PCOB memory-cue list (empty placeholder) + PCOB hot-cue list (empty placeholder) + +ANLZ0000.EXT — extended data; we write a minimal PPTH-only file so newer +firmware that opens .EXT as part of its load flow finds a valid container. +Color / 3-band waveforms (PWV4/PWV5/PWV6/PWV7) belong in .EXT/.2EX and +aren't generated yet. + +Field discoveries documented by byte-diffing maple's output against a real +rekordbox export at data/q1-rekordbox/. All multi-byte fields in ANLZ files are BIG-endian, unlike the pdb. """ +from __future__ import annotations + from pathlib import Path from struct import pack +from loguru import logger + +from maple.analyze import analyze_audio, load_analysis, save_analysis + +# Real rekordbox PMAI header carries three non-zero u4 markers in the bytes +# after the standard PMAI/len_header/len_file triple. Verified by hex-dump +# of data/q1-rekordbox/PIONEER/USBANLZ/P020/0002E10A/ANLZ0000.{DAT,EXT,2EX} +# — all three files share the same 16-byte trailer. +PMAI_HEADER_TRAILER = pack(">IIII", 0x00000001, 0x00010000, 0x00010000, 0x00000000) +ANLZ_HEADER_LEN = 28 # PMAI + 2 size u4s + the 16-byte trailer above + def _section_pth(path: str) -> bytes: """PPTH section: u4 len_path, then UTF-16BE bytes (null-terminated).""" utf16 = path.encode("utf-16-be") + b"\x00\x00" len_path = len(utf16) body = pack(">I", len_path) + utf16 - # Section header: 4 fourcc + u4 len_header + u4 len_tag fourcc = b"PPTH" - len_header = 16 # section header size (incl. fourcc + 2 u4s + len_path u4) - len_tag = 12 + len(body) # full section size + len_header = 16 # fourcc + 2 u4 size fields + len_path u4 + len_tag = 12 + len(body) return fourcc + pack(">II", len_header, len_tag) + body -def build_minimal_anlz(audio_path: str) -> bytes: - """Return bytes of a minimal ANLZ0000.DAT pointing at `audio_path`.""" - pth = _section_pth(audio_path) - header_len = 28 - file_len = header_len + len(pth) - header = b"PMAI" + pack(">II", header_len, file_len) - header += b"\x00" * (header_len - len(header)) - assert len(header) == header_len - return header + pth +def _section_pvbr() -> bytes: + """PVBR section: 400-entry VBR seek index. We write a zero-filled + placeholder — rekordbox accepts this for fixed-bitrate sources and it + matches the section's presence in real exports.""" + fourcc = b"PVBR" + len_header = 16 # prelude(12) + u4 unnamed0(4) + # 400 u4 seek index + 4 trailer bytes = 1604 bytes of variable data + # (verified by len_tag=1620 in the real Q1.1 export). + body_var = b"\x00" * 1604 + unnamed0 = b"\x00\x00\x00\x00" + len_tag = len_header + len(body_var) # 16 + 1604 = 1620 + return fourcc + pack(">II", len_header, len_tag) + unnamed0 + body_var -def write_anlz(audio_path: str, dest: Path) -> None: +def _section_pqtz(beats: list[tuple[int, int, int]]) -> bytes: + """PQTZ: BeatGridTag — per-beat (beat_number, tempo×100, time_ms). + + The CDJ's loaded-deck BPM readout, tempo slider and sync engine all + read from this section. Real rekordbox sets `_unnamed1=0x00080000` + (was 0 in maple's earlier writer — likely the cause of the deck-side + BPM display staying blank even when the grid was otherwise valid). + """ + fourcc = b"PQTZ" + len_header = 24 # prelude(12) + 3 u4 body-header fields + num_beats = len(beats) + body = pack(">III", 0x00000000, 0x00080000, num_beats) + for bn, tempo_x100, t_ms in beats: + body += pack(">HHI", bn, tempo_x100, t_ms) + len_tag = 12 + len(body) + return fourcc + pack(">II", len_header, len_tag) + body + + +def _section_pwav(preview: bytes) -> bytes: + """PWAV: WavePreviewTag — 400-byte blue/white touch-strip preview. + Each byte = (white_flag << 5) | height (bits 0-4).""" + fourcc = b"PWAV" + len_header = 20 # prelude(12) + len_data(4) + _unnamed1(4) + len_tag = len_header + len(preview) + return fourcc + pack(">IIII", len_header, len_tag, len(preview), 0x00010000) + preview + + +def _section_pwv2(tiny: bytes) -> bytes: + """PWV2: 100-byte "tiny" overview waveform. Same byte encoding as PWAV, + just at a coarser resolution. Real Q1.1 export has this section.""" + fourcc = b"PWV2" + len_header = 20 # prelude(12) + len_data(4) + _unnamed1(4) + len_tag = len_header + len(tiny) + return fourcc + pack(">IIII", len_header, len_tag, len(tiny), 0x00010000) + tiny + + +def _section_pcob(cue_list_type: int) -> bytes: + """PCOB: CueTag — cue/loop list. Real exports always include both + memory-cue (type=0) and hot-cue (type=1) sections, even when both are + empty. Body: u4 type, 2 bytes unnamed, u2 num_cues, u4 memory_count.""" + fourcc = b"PCOB" + len_header = 24 # prelude(12) + body fixed prefix(12) + body = pack(">I", cue_list_type) + b"\x00\x00" + pack(">HI", 0, 0) + len_tag = 12 + len(body) + return fourcc + pack(">II", len_header, len_tag) + body + + +def _section_pwv3(scroll: bytes) -> bytes: + """PWV3: WaveScrollTag — full-track mono scroll waveform. + + NOT written into ANLZ0000.DAT (real rekordbox places PWV3 in .EXT, not + .DAT). Kept here for reference / future .EXT generation.""" + fourcc = b"PWV3" + len_header = 24 + len_entry_bytes = 1 + len_entries = len(scroll) + len_tag = len_header + len(scroll) + return fourcc + pack(">IIIII", len_header, len_tag, len_entry_bytes, len_entries, 0x00960000) + scroll + + +def _bin_tiny_from_preview(preview: bytes, cols: int = 100) -> bytes: + """Derive the PWV2 tiny envelope from PWAV's 400-byte preview by + re-binning groups of 4 columns to their per-bin max.""" + if not preview: + return b"" + chunk = max(1, len(preview) // cols) + out = bytearray(cols) + for i in range(cols): + slc = preview[i * chunk : (i + 1) * chunk] if i < cols - 1 else preview[i * chunk:] + if slc: + # Per-byte max ignoring the white-flag bit, then put it back at + # the corresponding compressed amplitude. + heights = [b & 0x1F for b in slc] + out[i] = max(heights) & 0x1F + return bytes(out) + + +def _pmai_header(file_len: int) -> bytes: + """28-byte PMAI header — fourcc + sizes + the canonical trailer that + rekordbox writes (was zero-padded in maple's earlier output).""" + return b"PMAI" + pack(">II", ANLZ_HEADER_LEN, file_len) + PMAI_HEADER_TRAILER + + +def build_dat(rb_path: str, analysis: dict | None) -> bytes: + """Build ANLZ0000.DAT — the file every CDJ reads on track load. + + Section order matches rekordbox's own output: PPTH, PVBR, PQTZ, PWAV, + PWV2, PCOB(memory), PCOB(hot). + """ + sections = [_section_pth(rb_path), _section_pvbr()] + if analysis: + if analysis.get("beats"): + sections.append(_section_pqtz(analysis["beats"])) + preview = analysis.get("preview") or b"" + if preview: + sections.append(_section_pwav(preview)) + sections.append(_section_pwv2(_bin_tiny_from_preview(preview))) + sections.append(_section_pcob(0)) # memory cues (empty) + sections.append(_section_pcob(1)) # hot cues (empty) + body = b"".join(sections) + return _pmai_header(ANLZ_HEADER_LEN + len(body)) + body + + +def build_ext(rb_path: str) -> bytes: + """Build a minimal ANLZ0000.EXT — PPTH-only. + + Real rekordbox .EXT carries color waveforms and song-structure + sections; we don't generate those yet. A bare .EXT file is enough for + firmware that opens it as part of the standard track-load flow.""" + body = _section_pth(rb_path) + return _pmai_header(ANLZ_HEADER_LEN + len(body)) + body + + +def build_anlz(rb_path: str, source_audio: Path | None = None) -> bytes: + """Compatibility wrapper for callers that want the .DAT bytes only.""" + analysis: dict | None = None + if source_audio is not None: + try: + analysis = analyze_audio(source_audio) + except Exception as e: + logger.warning("audio analysis failed for {}: {}", source_audio.name, e) + return build_dat(rb_path, analysis) + + +def build_minimal_anlz(rb_path: str) -> bytes: + """PPTH-only-plus-empties-DAT. Used when no source audio is available.""" + return build_dat(rb_path, analysis=None) + + +def _isrc_for(path: Path) -> str | None: + """Best-effort ISRC read from a file tag (mp3/flac/m4a).""" + try: + from mutagen import File as MutagenFile + audio = MutagenFile(path, easy=True) + if audio is None or audio.tags is None: + return None + v = audio.tags.get("isrc") + if not v: + return None + return str(v[0]).strip() or None + except Exception: + return None + + +def write_anlz( + rb_path: str, + dest: Path, + source_audio: Path | None = None, + analysis_cache_dir: Path | None = None, +) -> dict | None: + """Write ANLZ0000.DAT (and a minimal ANLZ0000.EXT alongside). + + `dest` is the .DAT path. The .EXT companion is written next to it + automatically — every real CDJ export ships both files. + + Returns the analysis dict (cache hit or fresh aubio run) or None when + no source audio was given. + """ dest.parent.mkdir(parents=True, exist_ok=True) - dest.write_bytes(build_minimal_anlz(audio_path)) + analysis: dict | None = None + cache_hit = False + if source_audio is not None: + isrc = _isrc_for(source_audio) if analysis_cache_dir else None + if isrc and analysis_cache_dir: + cached = load_analysis(analysis_cache_dir, isrc) + if cached is not None: + analysis = cached + cache_hit = True + logger.info("anlz: cache hit for {} (isrc {})", source_audio.name, isrc) + if analysis is None: + try: + analysis = analyze_audio(source_audio) + except Exception as e: + logger.warning("audio analysis failed for {}: {}", source_audio.name, e) + analysis = None + # Populate the cache on every successful fresh analysis so subsequent + # exports (and collection runs) can skip the ffmpeg+aubio pass. The + # cache is the single source of truth — populated by whichever + # pipeline (`just collection` or `just export`) sees a track first. + if analysis is not None and not cache_hit and analysis_cache_dir is not None and isrc: + try: + cp = save_analysis(analysis_cache_dir, isrc, analysis) + if cp is not None: + logger.info("anlz: cached analysis for {} → {}", source_audio.name, cp) + except Exception as e: + logger.warning("could not write analysis cache for {}: {}", source_audio.name, e) + + dest.write_bytes(build_dat(rb_path, analysis)) + # .EXT sister file. Newer firmware (esp. CDJ-3000) opens this during the + # track-load flow; absence of the file has been observed to silently + # disable the loaded-deck BPM/waveform display even when .DAT is intact. + ext_path = dest.with_suffix(".EXT") + ext_path.write_bytes(build_ext(rb_path)) + return analysis diff --git a/maple-py/src/maple/cli.py b/maple-py/src/maple/cli.py index 859c01b..87072d6 100644 --- a/maple-py/src/maple/cli.py +++ b/maple-py/src/maple/cli.py @@ -1,34 +1,116 @@ """maple CLI: scan a directory of audio + m3u8 playlists and build a rekordbox USB.""" import argparse import os +import re import sys +import time +from collections import defaultdict from concurrent.futures import ThreadPoolExecutor from datetime import date from pathlib import Path +from loguru import logger + from maple.anlz import write_anlz +from maple.artwork import _bucket as _artwork_bucket from maple.artwork import extract_artwork from maple.convert import convert_flac_to_mp3, verify_duration from maple.copy import stage_audio, staged_path +from maple.log import configure as configure_logging from maple.meta import read_tags from maple.model import Album, Artist, Artwork, Database, Genre, PlaylistEntry, PlaylistNode, Track +from maple.enrich import enrich_from_manifest, load_spotify_manifest, write_bpm_tag from maple.pdb.writer import write_pdb from maple.scan import scan_audio, scan_playlists +from maple.validate import TrackIssue, log_issues, validate_track + +# Genre tag delimiters seen across our sources: Spotify-derived uses +# ';', European labels often use ',', some store '/'-separated lists. +_GENRE_SPLIT_RE = re.compile(r"\s*[;,/]\s*") -def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Database: +def split_genre_tags(raw: str) -> list[str]: + """Tokenise a possibly multi-value genre string into normalised parts. + + Each token's casing is preserved as-given. Empty parts are dropped. + Duplicates within the same source string are collapsed (first wins). + """ + if not raw: + return [] + seen: set[str] = set() + out: list[str] = [] + for token in _GENRE_SPLIT_RE.split(raw): + t = token.strip() + if not t: + continue + key = t.casefold() + if key in seen: + continue + seen.add(key) + out.append(t) + return out + + +def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> tuple[Database, set[Path]]: + """Build the rekordbox USB layout under `usb_root`. + + Returns the populated `Database` plus the set of absolute paths that + were written (audio files, ANLZ files, artwork JPEGs). The caller adds + the pdb path and calls `prune_stale` to remove anything else under + `usb_root` left over from a previous export. + """ db = Database() today = date.today().isoformat() artist_index: dict[str, int] = {} album_index: dict[tuple[int, str], int] = {} + # Key by casefolded name so "Dub Techno" and "dub techno" collapse to one row. genre_index: dict[str, int] = {} + # The user-facing genre→tracks map. Populated as tracks are processed. + genre_to_tracks: dict[str, list[int]] = defaultdict(list) track_by_src: dict[Path, int] = {} artwork_cache: dict[str, int] = {} + written: set[Path] = set() audio_files = scan_audio(input_dir) + logger.info("scanned {}: {} audio file(s)", input_dir, len(audio_files)) + # ---------- Spotify enrichment (symmetric with `just collection`) -------- # + # Per the validation convention: both pipelines must be capable of + # bringing a track to "valid" state. Validate first; only files that + # actually need it pay the enrichment cost (mutagen tag rewrite + + # possible artwork fetch). Repeated exports of an already-valid + # collection skip the enrichment loop entirely. + by_key, by_isrc = load_spotify_manifest(input_dir / "_metadata") + cache_dir = input_dir / "_analysis" + if by_key or by_isrc: + logger.info( + "loaded spotify manifest: {} (artist,title) keys, {} ISRCs", + len(by_key), len(by_isrc), + ) + enriched = 0 + skipped = 0 + for p in audio_files: + if not validate_track(p, analysis_cache_dir=cache_dir): + skipped += 1 + continue + if enrich_from_manifest(p, by_key, by_isrc, analysis_cache_dir=cache_dir) is not None: + enriched += 1 + logger.info( + "enrichment: {} enriched, {} already valid (skipped)", + enriched, skipped, + ) + else: + logger.warning( + "no spotify manifest at {} — tracks won't be Spotify-enriched. " + "Run `just collection` first to fetch + persist the manifest.", + input_dir / "_metadata" / "spotify-tracks.json", + ) + + t = time.perf_counter() initial_meta = {p: read_tags(p) for p in audio_files} + logger.debug("read tags in {:.2f}s", time.perf_counter() - t) + flac_jobs: list[tuple[Path, Path]] = [] for original in audio_files: if original.suffix.lower() == ".flac": @@ -38,15 +120,32 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Datab def _job(pair: tuple[Path, Path]) -> tuple[Path, Path, bool]: src, dst = pair - return src, dst, convert_flac_to_mp3(src, dst) + t0 = time.perf_counter() + converted = convert_flac_to_mp3(src, dst) + return src, dst, converted, time.perf_counter() - t0 - workers = max(1, os.cpu_count() or 1) - with ThreadPoolExecutor(max_workers=workers) as ex: - for src, dst, converted in ex.map(_job, flac_jobs): - print(f"{'convert' if converted else 'cached '} {src.relative_to(input_dir)}") - verify_duration(src, dst) + if flac_jobs: + workers = max(1, os.cpu_count() or 1) + logger.info("converting {} flac(s) → mp3 v0 (workers={})", len(flac_jobs), workers) + t = time.perf_counter() + n_converted = 0 + with ThreadPoolExecutor(max_workers=workers) as ex: + for i, (src, dst, converted, dt) in enumerate(ex.map(_job, flac_jobs), 1): + tag = "convert" if converted else "cached " + logger.info("[{}/{}] {} {} ({:.1f}s)", i, len(flac_jobs), tag, src.relative_to(input_dir), dt) + if converted: + n_converted += 1 + verify_duration(src, dst) + logger.info( + "flac→mp3 pass: {} converted, {} cached in {:.1f}s total", + n_converted, len(flac_jobs) - n_converted, time.perf_counter() - t, + ) - for original in audio_files: + logger.info("building track table ({} files)", len(audio_files)) + build_t0 = time.perf_counter() + validation_issues: list[TrackIssue] = [] + for track_idx, original in enumerate(audio_files, 1): + track_t0 = time.perf_counter() meta = initial_meta[original] artist_id = artist_index.get(meta.artist) if artist_id is None: @@ -66,19 +165,50 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Datab meta = read_tags(dest_abs) else: dest_abs, file_path = stage_audio(original, usb_root, meta.artist, meta.album) + written.add(dest_abs.resolve()) + + # Re-enrich the STAGED file too — `stage_audio` and the FLAC→MP3 + # converter cache by hash, so a cached dest may still carry stale + # tags from a previous export. Skip when the staged file already + # passes validation so re-exports of an unchanged collection don't + # re-mutate every track. + if (by_key or by_isrc) and validate_track(dest_abs, analysis_cache_dir=cache_dir): + if enrich_from_manifest(dest_abs, by_key, by_isrc, analysis_cache_dir=cache_dir) is not None: + meta = read_tags(dest_abs) next_artwork_id = len(db.artwork) + 1 artwork_id, art_file_path = extract_artwork(dest_abs, usb_root, next_artwork_id, artwork_cache) if art_file_path is not None: db.artwork.append(Artwork(id=artwork_id, path=art_file_path)) + # Track artwork paths for the prune sweep — both this run's writes + # and any artwork_cache hits (where extract_artwork returned None + # for art_file_path because the jpeg already exists from a prior + # iteration). _bucket() is the same id-to-folder mapping artwork.py + # uses, so this rebuilds the absolute on-disk path exactly. + if artwork_id: + art_bucket = _artwork_bucket(artwork_id) + for fname in (f"a{artwork_id}.jpg", f"a{artwork_id}_m.jpg"): + written.add((usb_root / "PIONEER" / "Artwork" / art_bucket / fname).resolve()) - genre_id = 0 - if meta.genre: - genre_id = genre_index.get(meta.genre, 0) - if genre_id == 0: - genre_id = len(db.genres) + 1 - genre_index[meta.genre] = genre_id - db.genres.append(Genre(id=genre_id, name=meta.genre)) + # Genre: split multi-value tag strings ("dub techno;minimal techno", + # "Drum & Bass, Électronique", etc.) into individual genre rows so the + # CDJ's Genre filter is usable. The track FK points at the FIRST + # (most-specific) token; every other token still gets its own row in + # the pdb's genre table so it appears in the browse list, and we + # record the {genre→track_ids} map so we can log + reuse it later. + primary_genre_id = 0 + tokens = split_genre_tags(meta.genre) + for n, token in enumerate(tokens): + key = token.casefold() + gid = genre_index.get(key) + if gid is None: + gid = len(db.genres) + 1 + genre_index[key] = gid + db.genres.append(Genre(id=gid, name=token)) + if n == 0: + primary_genre_id = gid + genre_to_tracks[token].append(len(db.tracks) + 1) # this track's id-to-be + genre_id = primary_genre_id track_id = len(db.tracks) + 1 # Reference exports always populate analyze_path even when the @@ -95,7 +225,29 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Datab bucket = f"P{(h >> 20) & 0x7F:02X}" analyze_id = f"{h & 0xFFFFFFFF:08X}" analyze_path = f"/PIONEER/USBANLZ/{bucket}/{analyze_id}/ANLZ0000.DAT" - write_anlz(file_path, usb_root / analyze_path.lstrip("/")) + anlz_abs = (usb_root / analyze_path.lstrip("/")).resolve() + # `_analysis/` sits at the input collection root; ISRC-keyed cache + # populated during `just collection`. write_anlz reads it before + # falling back to a fresh ffmpeg+aubio pass. + analysis = write_anlz( + file_path, anlz_abs, source_audio=dest_abs, + analysis_cache_dir=input_dir / "_analysis", + ) + written.add(anlz_abs) + # .EXT sister file is written automatically next to the .DAT. + written.add(anlz_abs.with_suffix(".EXT")) + # Prefer the analytically detected BPM over the file tag when we have one. + # CDJ browse panel reads pdb.tempo; loaded-deck readout reads the PQTZ + # we just wrote — keeping the two in sync avoids the "120.0 in browse, + # blank on deck" UX bug. + if analysis and analysis.get("bpm"): + tempo_x100 = round(analysis["bpm"] * 100) + # Also write the bpm TAG to the file (symmetric with `just + # collection`'s _write_bpm_tag). overwrite=False so any + # pre-existing label-set BPM is preserved. + write_bpm_tag(dest_abs, analysis["bpm"], overwrite=False) + else: + tempo_x100 = meta.tempo db.tracks.append(Track( id=track_id, title=meta.title, @@ -110,7 +262,7 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Datab duration=meta.duration, track_number=meta.track_number, year=meta.year, - tempo=meta.tempo, + tempo=tempo_x100, date_added=today, analyze_path=analyze_path, analyze_date=today, @@ -120,9 +272,61 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Datab )) track_by_src[original.resolve()] = track_id + # Cross-cutting validation: same checks as `just collection`. The + # input audio_path is the staged file under data/usb/ so its tags + # are what end up on the player. analysis cache lives at the source + # collection root (input_dir/_analysis/.json). + track_issues = validate_track( + dest_abs, + analysis_cache_dir=input_dir / "_analysis", + ) + validation_issues.extend(track_issues) + + track_dt = time.perf_counter() - track_t0 + bpm_str = f"{analysis['bpm']:.1f} bpm" if (analysis and analysis.get("bpm")) else f"tag {tempo_x100/100:.1f} bpm" + issue_tag = f" [{len(track_issues)} issue(s)]" if track_issues else "" + logger.info( + "[{}/{}] {} - {} ({}, {:.1f}s){}", + track_idx, len(audio_files), + meta.artist, meta.title, + bpm_str, track_dt, issue_tag, + ) + logger.info( + "track table built in {:.1f}s ({} tracks, {} artists, {} albums, " + "{} artwork, {} genres)", + time.perf_counter() - build_t0, + len(db.tracks), len(db.artists), len(db.albums), + len(db.artwork), len(db.genres), + ) + # Genre map summary: largest buckets first; helps spot mistagged tracks. + if genre_to_tracks: + logger.info("genre→tracks map ({} genres):", len(genre_to_tracks)) + for genre, track_ids in sorted( + genre_to_tracks.items(), key=lambda kv: (-len(kv[1]), kv[0]) + ): + logger.info(" {:<32s} {:>3} track(s)", genre[:32], len(track_ids)) + no_genre = sum(1 for t in db.tracks if t.genre_id == 0) + if no_genre: + logger.warning("{} of {} tracks have no genre tag", no_genre, len(db.tracks)) + + # Emit the validation report. Same TrackIssue shape as `just collection`, + # surfaced through loguru so the user gets visibility plus a roll-up at + # the end. + if validation_issues: + logger.warning( + "{} validation issue(s) across {} track(s)", + len(validation_issues), + len({(i.artist, i.title) for i in validation_issues}), + ) + log_issues(validation_issues, report=False) + else: + logger.info("validation: all {} tracks passed", len(audio_files)) + if not with_playlists: - return db + return db, written, validation_issues playlists = scan_playlists(input_dir) + if playlists: + logger.info("scanning {} playlist(s)", len(playlists)) for sort_order, (name, paths) in enumerate(playlists.items(), start=1): playlist_id = len(db.playlist_nodes) + 1 db.playlist_nodes.append(PlaylistNode( @@ -132,18 +336,58 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Datab name=name, is_folder=False, )) + resolved = 0 for entry_index, p in enumerate(paths, start=1): tid = track_by_src.get(p.resolve()) if tid is None: - print(f"warning: playlist {name!r} references missing file {p}", file=sys.stderr) + logger.warning("playlist {!r} references missing file {}", name, p) continue db.playlist_entries.append(PlaylistEntry( playlist_id=playlist_id, track_id=tid, entry_index=entry_index, )) + resolved += 1 + logger.info(" {}: {} entries ({} resolved, {} missing)", + name, len(paths), resolved, len(paths) - resolved) - return db + return db, written, validation_issues + + +def prune_stale(usb_root: Path, kept: set[Path]) -> tuple[int, int]: + """Delete every file under `usb_root` not in `kept`, then drop empty dirs. + + Returns (files_removed, dirs_removed). Operates only under `usb_root`, + so anything outside is untouched. The `kept` set is expected to hold + fully-resolved absolute paths. + """ + files_removed = 0 + for p in usb_root.rglob("*"): + if not p.is_file(): + continue + if p.resolve() in kept: + continue + try: + p.unlink() + files_removed += 1 + except OSError as e: + logger.warning("failed to remove {}: {}", p, e) + + # Bottom-up empty-dir removal: deepest first so each parent gets a + # chance to become empty once its children are gone. + dirs = sorted( + (d for d in usb_root.rglob("*") if d.is_dir()), + key=lambda d: len(d.parts), + reverse=True, + ) + dirs_removed = 0 + for d in dirs: + try: + d.rmdir() + dirs_removed += 1 + except OSError: + pass # not empty; expected for the directories we still want + return files_removed, dirs_removed def main(argv: list[str] | None = None) -> int: @@ -151,16 +395,41 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("input_dir", nargs="?", type=Path, default=Path("./data/input")) parser.add_argument("output_root", nargs="?", type=Path, default=Path("./data/usb")) parser.add_argument("--no-playlists", action="store_true", help="Skip m3u8 playlist scanning") + parser.add_argument("--no-prune", action="store_true", help="Skip the post-build sweep of stale files") + parser.add_argument("--strict", action="store_true", help="Fail with non-zero exit if any track fails validation") args = parser.parse_args(argv) + configure_logging() if not args.input_dir.is_dir(): parser.error(f"input_dir {args.input_dir} is not a directory") - db = build(args.input_dir, args.output_root, with_playlists=not args.no_playlists) + overall_t0 = time.perf_counter() + logger.info("export: {} → {}", args.input_dir, args.output_root) + db, written, validation_issues = build(args.input_dir, args.output_root, with_playlists=not args.no_playlists) pdb_path = args.output_root / "PIONEER" / "rekordbox" / "export.pdb" write_pdb(db, pdb_path) + written.add(pdb_path.resolve()) - print(f"wrote {len(db.tracks)} tracks, {len(db.artists)} artists, {len(db.albums)} albums, " - f"{len(db.playlist_nodes)} playlists ({len(db.playlist_entries)} entries) to {pdb_path}") + if not args.no_prune: + t = time.perf_counter() + n_files, n_dirs = prune_stale(args.output_root, written) + if n_files or n_dirs: + logger.info( + "pruned {} stale file(s) + {} empty dir(s) in {:.1f}s", + n_files, n_dirs, time.perf_counter() - t, + ) + else: + logger.debug("nothing to prune ({:.1f}s)", time.perf_counter() - t) + + logger.info( + "done in {:.1f}s: {} tracks, {} artists, {} albums, " + "{} playlists ({} entries) → {}", + time.perf_counter() - overall_t0, + len(db.tracks), len(db.artists), len(db.albums), + len(db.playlist_nodes), len(db.playlist_entries), pdb_path, + ) + if args.strict and validation_issues: + logger.error("--strict: refusing to succeed with {} validation issue(s)", len(validation_issues)) + return 1 return 0 diff --git a/maple-py/src/maple/collection/__init__.py b/maple-py/src/maple/collection/__init__.py index a62d211..8019e6c 100644 --- a/maple-py/src/maple/collection/__init__.py +++ b/maple-py/src/maple/collection/__init__.py @@ -21,7 +21,18 @@ Env (from .env, loaded by just): MAPLE_SPOTIFY_CLIENT_ID, MAPLE_SPOTIFY_CLIENT_SECRET (spotify.py reads) SOCKSEEK_USER, SOCKSEEK_PASS (forwarded to sockseek) """ -from maple.collection.core import Collection, TrackIssue -from maple.collection.index import CollectionIndex - +# Re-exports are lazy via __getattr__ so importing `maple.collection.index` +# (or any other submodule) doesn't drag the whole package — `maple.validate` +# depends on `maple.collection.index` and would otherwise circular-import +# through `maple.collection.core` → `maple.validate` → here. __all__ = ["Collection", "CollectionIndex", "TrackIssue"] + + +def __getattr__(name): + if name == "Collection" or name == "TrackIssue": + from maple.collection.core import Collection, TrackIssue + return {"Collection": Collection, "TrackIssue": TrackIssue}[name] + if name == "CollectionIndex": + from maple.collection.index import CollectionIndex + return CollectionIndex + raise AttributeError(name) diff --git a/maple-py/src/maple/collection/cli.py b/maple-py/src/maple/collection/cli.py index 309d955..0f773d0 100644 --- a/maple-py/src/maple/collection/cli.py +++ b/maple-py/src/maple/collection/cli.py @@ -16,6 +16,7 @@ from loguru import logger from maple.collection.core import Collection, TrackIssue from maple.collection.index import first_artist, norm +from maple.enrich import save_spotify_manifest from maple.log import configure as configure_logging from maple.spotify import fetch_playlists @@ -55,6 +56,9 @@ async def main_async(argv: list[str]) -> int: playlists = await fetch_playlists(args.ids) total = sum(len(p["tracks"]) for p in playlists) logger.info("fetched {} playlist(s), {} tracks", len(playlists), total) + # Persist the full set of Spotify tracks so `just export` (and any + # later `just collection` run) can enrich tags without re-fetching. + save_spotify_manifest(collection.metadata_dir, playlists) collection.refresh_index() logger.info( diff --git a/maple-py/src/maple/collection/core.py b/maple-py/src/maple/collection/core.py index 08b2ffa..494c3ff 100644 --- a/maple-py/src/maple/collection/core.py +++ b/maple-py/src/maple/collection/core.py @@ -23,6 +23,10 @@ import aiohttp from loguru import logger from mutagen import File as MutagenFile +from maple.analyze import analyze_audio, detect_bpm, save_analysis +from maple.enrich import write_spotify_tags as _shared_write_spotify_tags +from maple.validate import TrackIssue as _ValidateTrackIssue +from maple.validate import validate_track as _validate_track from maple.collection.index import ( CollectionIndex, first_artist, @@ -42,13 +46,14 @@ from maple.collection.validate import ( ) from maple.scan import AUDIO_EXTS +BPM_DIVERGENCE_WARN = 0.05 # warn when aubio vs existing tag differs >5% -@dataclass -class TrackIssue: - playlist: str - artist: str - title: str - reason: str + +# Re-export the shared validator's TrackIssue so callers that imported it +# from maple.collection.core continue to work. There is one TrackIssue +# definition in the codebase — `maple.validate.TrackIssue` — and both the +# `just collection` and `just export` flows produce/consume the same shape. +TrackIssue = _ValidateTrackIssue _M3U_BAD = re.compile(r"[\\/\x00]+") @@ -95,10 +100,17 @@ class Collection: def report_path(self) -> Path: return self.metadata_dir / "collection-report.txt" + @property + def analysis_dir(self) -> Path: + """`/_analysis/.json` — cached aubio output keyed on ISRC, + consumed by `just export` to skip the decode+detect pass.""" + return self.root / "_analysis" + def ensure_dirs(self) -> None: self.root.mkdir(parents=True, exist_ok=True) self.metadata_dir.mkdir(parents=True, exist_ok=True) self.playlists_dir.mkdir(parents=True, exist_ok=True) + self.analysis_dir.mkdir(parents=True, exist_ok=True) # --- startup validation ----------------------------------------------- # @@ -405,58 +417,38 @@ class Collection: return issues def write_spotify_tags(self, track: dict, path: Path) -> None: - """Overwrite the file's tags with the Spotify values. + """Delegate to the shared `maple.enrich.write_spotify_tags` so the + collection-side and export-side writes are byte-for-byte identical + and can't drift.""" + _shared_write_spotify_tags(track, path) - Spotify is canonical: title, artist (joined ", "), album, date, - genre(s), and ISRC are written. Missing Spotify values leave the - existing tag untouched. - """ + def _existing_bpm(self, path: Path) -> float | None: + """Return the parsed `bpm` tag value, or None if absent/invalid.""" try: audio = MutagenFile(path, easy=True) - except Exception as e: - logger.warning("write_spotify_tags: cannot open {}: {}", path.name, e) - return - if audio is None: - return - if audio.tags is None: - try: - audio.add_tags() - except Exception as e: - logger.warning("write_spotify_tags: cannot init tags on {}: {}", path.name, e) - return - - title = (track.get("title") or "").strip() - if title: - audio.tags["title"] = title - artists = track.get("artists") or [] - if artists: - audio.tags["artist"] = ", ".join(artists) - album = (track.get("album") or "").strip() - if album: - audio.tags["album"] = album - date = (track.get("release_date") or "").strip() - if date: - try: - audio.tags["date"] = date - except (KeyError, ValueError) as e: - logger.debug("date tag write skipped on {}: {}", path.name, e) - genres = track.get("genres") or [] - if genres: - try: - audio.tags["genre"] = list(genres) - except (KeyError, ValueError) as e: - logger.debug("genre tag write skipped on {}: {}", path.name, e) - isrc = (track.get("isrc") or "").strip() - if isrc: - try: - audio.tags["isrc"] = isrc - except (KeyError, ValueError) as e: - logger.debug("isrc tag write skipped on {}: {}", path.name, e) + if audio is None or audio.tags is None: + return None + v = audio.tags.get("bpm") + if not v: + return None + return float(v[0]) + except (ValueError, TypeError, Exception): + return None + def _write_bpm_tag(self, path: Path, bpm: float) -> None: try: + audio = MutagenFile(path, easy=True) + if audio is None: + return + if audio.tags is None: + try: + audio.add_tags() + except Exception: + return + audio.tags["bpm"] = f"{bpm:.2f}" audio.save() except Exception as e: - logger.warning("write_spotify_tags: save failed for {}: {}", path.name, e) + logger.warning("failed to write bpm to {}: {}", path.name, e) def process_track( self, @@ -464,12 +456,14 @@ class Collection: track: dict, path: Path, ) -> list[TrackIssue]: - """Reconcile a single downloaded file against Spotify. + """Reconcile a single downloaded file against Spotify + analyse it. Steps: 1. ffprobe sanity check (one issue + early return on failure). 2. Compare every relevant field; collect mismatch issues. 3. Write Spotify's values back into the file's tags. + 4. If the file has no `bpm` tag yet, run aubio BPM detection + and write the analytical value. Existing tags are left alone. Idempotent within a run — paths already processed are no-ops. """ @@ -477,14 +471,49 @@ class Collection: return [] self._reconciled.add(path) - sp_artist = ", ".join(track.get("artists") or []) - sp_title = track.get("title") or "" - if not ffprobe_ok(path): - return [TrackIssue(playlist_name, sp_artist, sp_title, f"ffprobe failed: {path}")] - - issues = self.compare_against_spotify(playlist_name, track, path) + # Source-of-truth tag writes first: ensures the file mirrors Spotify + # before validation reads its tags back. Mismatches caught here are + # written to the report regardless of the validator's subsequent + # findings (the validator may dedup them if Spotify still disagrees). self.write_spotify_tags(track, path) - return issues + + # Single audio analysis pass: BPM + beat grid + waveform envelopes. + # Cached under collection/_analysis/.json so `just export` + # reads the bytes back instead of redoing ffmpeg+aubio. + isrc = (track.get("isrc") or "").strip() + existing_bpm = self._existing_bpm(path) + try: + analysis = analyze_audio(path) + except Exception as e: + logger.warning("analyze_audio failed on {}: {}", path.name, e) + analysis = None + if analysis and analysis.get("bpm"): + if existing_bpm is None: + self._write_bpm_tag(path, analysis["bpm"]) + logger.info("{}: bpm {:.2f} (aubio)", path.name, analysis["bpm"]) + else: + logger.debug( + "{}: bpm {:.2f} (existing tag, kept; aubio measured {:.2f})", + path.name, existing_bpm, analysis["bpm"], + ) + if isrc: + cp = save_analysis(self.analysis_dir, isrc, analysis) + logger.info("cached analysis for {}: {} → {}", isrc, path.name, cp) + else: + logger.warning( + "skipping analysis cache for {}: no ISRC in spotify track ({} - {})", + path.name, ", ".join(track.get("artists") or []), track.get("title") or "?", + ) + + # Full cross-cutting validation pass. Runs the SAME checks as + # `just export`, so a clean run here means the track will export + # without issues — and vice versa. + return _validate_track( + path, + spotify_track=track, + analysis_cache_dir=self.analysis_dir, + playlist=playlist_name, + ) # --- validation -------------------------------------------------------- # diff --git a/maple-py/src/maple/enrich.py b/maple-py/src/maple/enrich.py new file mode 100644 index 0000000..f1a17c9 --- /dev/null +++ b/maple-py/src/maple/enrich.py @@ -0,0 +1,307 @@ +"""Spotify metadata enrichment shared by `just collection` and `just export`. + +Per the project convention: a track is "valid" iff every export criterion +holds (artwork, tags, analysis). BOTH pipelines must be capable of +*bringing* a track to that state — validation isn't just reporting, +it's repair. + +`just collection`: + 1. Fetches Spotify playlists. + 2. Persists every track to `/_metadata/spotify-tracks.json`. + 3. Walks the collection, calls `write_spotify_tags` for each match. + +`just export`: + 1. Loads the manifest written by `just collection`. + 2. Walks `data/usb/`'s staged files. For any file missing canonical + tags (typically ISRC), looks up its Spotify entry via the manifest + and calls `write_spotify_tags` BEFORE validation runs. + +The shared write function lives here so the two pipelines are +byte-for-byte consistent in what they leave on disk. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from loguru import logger +from mutagen import File as MutagenFile +from mutagen.flac import FLAC, Picture +from mutagen.id3 import APIC, ID3, ID3NoHeaderError +from mutagen.mp4 import MP4, MP4Cover + +from maple.collection.index import first_artist, norm +from maple.collection.validate import embed_art, has_embedded_art + +MANIFEST_NAME = "spotify-tracks.json" + + +# --------------------------------------------------------------------------- # +# Manifest persistence # +# --------------------------------------------------------------------------- # + + +def save_spotify_manifest(metadata_dir: Path, playlists: list[dict]) -> Path: + """Persist Spotify-fetched playlists so `just export` can read them later. + + Stored as a single JSON blob at `/spotify-tracks.json`. + Replaces any prior manifest atomically (tmp + rename). + """ + metadata_dir.mkdir(parents=True, exist_ok=True) + target = metadata_dir / MANIFEST_NAME + tmp = target.with_suffix(target.suffix + ".tmp") + tmp.write_text( + json.dumps({"playlists": playlists}, indent=2, ensure_ascii=False, default=str) + ) + tmp.replace(target) + n = sum(len(p.get("tracks") or []) for p in playlists) + logger.info("wrote spotify manifest: {} tracks across {} playlist(s) → {}", + n, len(playlists), target) + return target + + +def load_spotify_manifest( + metadata_dir: Path, +) -> tuple[dict[tuple[str, str], dict], dict[str, dict]]: + """Load the manifest written by `just collection`. + + Returns `(by_key, by_isrc)` where `by_key` is indexed by + `(first_artist_norm, title_norm)` and `by_isrc` is indexed by the + uppercased ISRC. Empty dicts if the manifest is absent/unreadable. + """ + target = metadata_dir / MANIFEST_NAME + if not target.exists(): + return {}, {} + try: + data = json.loads(target.read_text()) + except Exception as e: + logger.warning("could not read spotify manifest at {}: {}", target, e) + return {}, {} + by_key: dict[tuple[str, str], dict] = {} + by_isrc: dict[str, dict] = {} + for pl in data.get("playlists", []): + for t in pl.get("tracks", []): + artist = ", ".join(t.get("artists") or []) + k = (norm(first_artist(artist)), norm(t.get("title") or "")) + by_key.setdefault(k, t) + isrc = (t.get("isrc") or "").strip().upper() + if isrc: + by_isrc.setdefault(isrc, t) + return by_key, by_isrc + + +def find_spotify_track( + path: Path, + by_key: dict[tuple[str, str], dict], + by_isrc: dict[str, dict], +) -> dict | None: + """Resolve a file to its Spotify track via ISRC tag, then (artist, title).""" + try: + audio = MutagenFile(path, easy=True) + except Exception: + return None + if audio is None or audio.tags is None: + return None + tags = audio.tags + + def first(k: str) -> str: + if hasattr(tags, "get"): + v = tags.get(k) + if v: + return str(v[0]) + return "" + + isrc = first("isrc").upper().strip() + if isrc and isrc in by_isrc: + return by_isrc[isrc] + artist = first("artist") + title = first("title") + if artist or title: + k = (norm(first_artist(artist)), norm(title)) + return by_key.get(k) + return None + + +# --------------------------------------------------------------------------- # +# Tag writing (the canonical write path) # +# --------------------------------------------------------------------------- # + + +def write_spotify_tags(track: dict, path: Path) -> bool: + """Overwrite the file's tags with the Spotify-canonical values. + + Both pipelines call this. Writes title, artist (`", ".join`), + album, date, genre (list), and ISRC via mutagen's easy interface. + Empty Spotify values leave the existing tag untouched. + + Returns True when at least one tag was written + saved. + """ + try: + audio = MutagenFile(path, easy=True) + except Exception as e: + logger.warning("write_spotify_tags: cannot open {}: {}", path.name, e) + return False + if audio is None: + return False + if audio.tags is None: + try: + audio.add_tags() + except Exception as e: + logger.warning("write_spotify_tags: cannot init tags on {}: {}", path.name, e) + return False + + wrote_any = False + title = (track.get("title") or "").strip() + if title: + audio.tags["title"] = title + wrote_any = True + artists = track.get("artists") or [] + if artists: + audio.tags["artist"] = ", ".join(artists) + wrote_any = True + album = (track.get("album") or "").strip() + if album: + audio.tags["album"] = album + wrote_any = True + date = (track.get("release_date") or "").strip() + if date: + try: + audio.tags["date"] = date + wrote_any = True + except (KeyError, ValueError) as e: + logger.debug("date tag write skipped on {}: {}", path.name, e) + genres = track.get("genres") or [] + if genres: + try: + audio.tags["genre"] = list(genres) + wrote_any = True + except (KeyError, ValueError) as e: + logger.debug("genre tag write skipped on {}: {}", path.name, e) + isrc = (track.get("isrc") or "").strip() + if isrc: + try: + audio.tags["isrc"] = isrc + wrote_any = True + except (KeyError, ValueError) as e: + logger.debug("isrc tag write skipped on {}: {}", path.name, e) + + if not wrote_any: + return False + try: + audio.save() + return True + except Exception as e: + logger.warning("write_spotify_tags: save failed for {}: {}", path.name, e) + return False + + +def write_bpm_tag(path: Path, bpm: float, *, overwrite: bool = False) -> bool: + """Write a `bpm` tag to the file. When `overwrite=False` (the default), + leaves any existing tag value alone — preserves human/label-set BPMs.""" + try: + audio = MutagenFile(path, easy=True) + except Exception as e: + logger.warning("write_bpm_tag: cannot open {}: {}", path.name, e) + return False + if audio is None: + return False + if audio.tags is None: + try: + audio.add_tags() + except Exception as e: + logger.warning("write_bpm_tag: cannot init tags on {}: {}", path.name, e) + return False + if not overwrite: + existing = audio.tags.get("bpm") if hasattr(audio.tags, "get") else None + if existing and (existing[0] if isinstance(existing, list) else existing): + return False # keep the existing tag + audio.tags["bpm"] = f"{bpm:.2f}" + try: + audio.save() + return True + except Exception as e: + logger.warning("write_bpm_tag: save failed for {}: {}", path.name, e) + return False + + +def download_jpeg(url: str, *, timeout: float = 15.0) -> bytes | None: + """Sync GET of a Spotify album_image_url. Returns the raw bytes or + None on failure. Used by `embed_artwork_from_track` — kept sync so + both `just collection`'s async loop and `just export`'s sync loop + can call into the same helper without contortions.""" + if not url: + return None + try: + import urllib.request + req = urllib.request.Request(url, headers={"User-Agent": "maple/1.0"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.read() + except Exception as e: + logger.warning("artwork fetch failed for {}: {}", url, e) + return None + + +def embed_artwork_from_track(track: dict, path: Path, *, force: bool = False) -> bool: + """Fetch the Spotify cover art for `track` and embed it into `path`. + + No-op when the file already has embedded art (unless `force=True`). + Skips silently when the track dict has no `album_image_url` (Spotify + occasionally returns tracks without images). + """ + if not force and has_embedded_art(path): + return False + url = (track.get("album_image_url") or "").strip() + if not url: + return False + data = download_jpeg(url) + if not data: + return False + try: + embed_art(path, data) + logger.info( + "embedded artwork for {} (from {})", + path.name, url.rsplit("/", 1)[-1][:40], + ) + return True + except Exception as e: + logger.warning("embed_art failed for {}: {}", path.name, e) + return False + + +def enrich_from_manifest( + path: Path, + by_key: dict[tuple[str, str], dict], + by_isrc: dict[str, dict], + analysis_cache_dir: Path | None = None, +) -> dict | None: + """Find this file's Spotify entry, bring it to "valid" state, return + the matched track dict (or None if no match). + + Steps: + 1. Write canonical text tags via `write_spotify_tags`. + 2. Fetch + embed album art if the file lacks it (`embed_artwork_from_track`). + 3. If `analysis_cache_dir` is supplied and the cache has an entry for + this track's ISRC, write the cached BPM as a tag. This lets the + enrich pass bring a track all the way to "validates" without a + separate aubio-analysis step. + """ + track = find_spotify_track(path, by_key, by_isrc) + if track is None: + return None + if write_spotify_tags(track, path): + logger.info( + "enriched {} from spotify (isrc={!r}, title={!r})", + path.name, (track.get("isrc") or "")[:12], (track.get("title") or "")[:32], + ) + embed_artwork_from_track(track, path) + + # Backfill BPM tag from the analysis cache when present. Avoids + # re-running aubio here — if the cache is cold, the per-track loop's + # write_anlz will populate it and write the tag in a later pass. + isrc = (track.get("isrc") or "").strip() + if analysis_cache_dir is not None and isrc: + from maple.analyze import load_analysis + cached = load_analysis(analysis_cache_dir, isrc) + if cached and cached.get("bpm"): + write_bpm_tag(path, cached["bpm"], overwrite=False) + return track diff --git a/maple-py/src/maple/pdb/writer.py b/maple-py/src/maple/pdb/writer.py index 97f2526..cb5e0bb 100644 --- a/maple-py/src/maple/pdb/writer.py +++ b/maple-py/src/maple/pdb/writer.py @@ -2,6 +2,8 @@ from pathlib import Path from struct import pack, unpack_from +from loguru import logger + from maple.model import ( DEFAULT_COLUMNS, STANDARD_COLORS, @@ -98,11 +100,21 @@ def _next_sequence(path: Path) -> int: def write_pdb(db: Database, path: str | Path) -> None: + logger.info( + "building pdb → {} (tracks={} artists={} albums={} genres={} " + "artwork={} playlists={} entries={})", + path, + len(db.tracks), len(db.artists), len(db.albums), len(db.genres), + len(db.artwork), len(db.playlist_nodes), len(db.playlist_entries), + ) # Refuse to emit a pdb with dangling foreign keys -- a track pointing at a # genre/artist/album/artwork row that doesn't exist makes the player follow # the FK into an empty table and freeze (freeze-class, like the deck hang). fk_errors = validate_referential_integrity(db) if fk_errors: + logger.error("FK validation failed with {} error(s)", len(fk_errors)) + for e in fk_errors: + logger.error(" {}", e) raise ValueError("dangling foreign keys (would freeze the player):\n " + "\n ".join(fk_errors)) # Layout (matching rekordbox): page 0 is the file header. Pages 1..2*N are @@ -215,6 +227,7 @@ def write_pdb(db: Database, path: str | Path) -> None: b.next_page = free_sentinel for b in last_builders: pages_by_index[b.page_index] = b.finalize() + overflow_used = overflow_next - (1 + 2 * num_tables) # Materialise a small free pool of zero-filled pages and set next_unused PAST # it, mirroring a real export (e.g. data/MAPLE: 71 pages, next_unused=74 -> @@ -232,6 +245,10 @@ def write_pdb(db: Database, path: str | Path) -> None: next_unused_page = file_page_count + 2 sequence = _next_sequence(Path(path)) + logger.info( + "pdb layout: {} pages ({} reserved, {} overflow, {} free), sequence={}", + file_page_count, 2 * num_tables, overflow_used, FREE_POOL_PAGES, sequence, + ) file_header = b"" file_header += pack(" None: page = pages_by_index.get(idx, blank) assert len(page) == PAGE_SIZE f.write(page) + logger.info("wrote {} ({} bytes)", out_path, out_path.stat().st_size) diff --git a/maple-py/src/maple/track.py b/maple-py/src/maple/track.py new file mode 100644 index 0000000..02b86dd --- /dev/null +++ b/maple-py/src/maple/track.py @@ -0,0 +1,261 @@ +"""Canonical `Track` — the unit shared by `just collection` and `just export`. + +A *collection* is a sequence of `Track`s persisted under +`data/collection/`. An *export* reads a sequence of `Track`s and writes a +rekordbox USB at `data/usb/`. This dataclass is the bridge: any code that +talks about "a track" in maple should hold one of these. + +Field semantics: + - Identity-ish fields (`isrc`, `title`, `artists`, `album`) start populated + from whichever source first knows the track — Spotify, a file's existing + tags, or a manual entry. + - Spotify-only fields (`spotify_track_id`, `release_date`, …) stay empty + for tracks that didn't come through the Spotify path. + - On-disk fields (`path`, `bitrate`, …) are populated once the audio is + actually on disk somewhere. + - Analytical fields (`bpm`, `beats`, …) populate during the audio-analysis + pass and are cached under `/_analysis/.json`. + +The existing `maple.model.Track` is a *denormalised pdb row* — it carries +artist_id/album_id/genre_id FKs that only exist in the context of a +specific export. `Track.to_pdb_row(...)` projects this canonical Track +into that shape using caller-supplied FK indices. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from maple import model + + +@dataclass +class Track: + """Canonical track. Defaults intentionally permissive so partially-filled + instances flow through the pipeline; validation (`maple.validate`) + decides what counts as "complete enough to export".""" + + # ---- identity --------------------------------------------------------- # + isrc: str = "" + title: str = "" + artists: list[str] = field(default_factory=list) + album: str = "" + + # ---- Spotify-derived ------------------------------------------------- # + spotify_track_id: str = "" + spotify_artist_ids: list[str] = field(default_factory=list) + album_image_url: str = "" + release_date: str = "" # "YYYY", "YYYY-MM", or "YYYY-MM-DD" + release_date_precision: str = "" + genres: list[str] = field(default_factory=list) + track_number: int = 0 + disc_number: int = 0 + duration_ms: int = 0 + + # ---- on-disk audio --------------------------------------------------- # + path: Path | None = None # current best file location + file_size: int = 0 + bitrate: int = 0 # kbps + sample_rate: int = 0 # Hz + sample_depth: int = 0 # bits per sample + + # ---- analytical ------------------------------------------------------ # + bpm: float | None = None # median-of-intervals tempo + + # ---- provenance ------------------------------------------------------ # + source: str = "" # "spotify", "local", "bandcamp", … + + # ---- views ----------------------------------------------------------- # + + @property + def primary_artist(self) -> str: + return self.artists[0] if self.artists else "" + + @property + def display_name(self) -> str: + if not self.title: + return self.path.name if self.path else "" + if self.artists: + return f"{', '.join(self.artists)} - {self.title}" + return self.title + + @property + def primary_genre(self) -> str: + return self.genres[0] if self.genres else "" + + @property + def year(self) -> int: + """First 4 digits of release_date as an int (0 if absent/invalid).""" + if not self.release_date or len(self.release_date) < 4: + return 0 + try: + return int(self.release_date[:4]) + except ValueError: + return 0 + + @property + def duration_s(self) -> int: + return int(round(self.duration_ms / 1000)) if self.duration_ms else 0 + + # ---- ingress --------------------------------------------------------- # + + @classmethod + def from_spotify(cls, d: dict) -> "Track": + """Build from one row of `maple.spotify.fetch_playlist`'s output.""" + return cls( + isrc=(d.get("isrc") or "").strip(), + title=d.get("title") or "", + artists=list(d.get("artists") or []), + album=d.get("album") or "", + spotify_track_id=d.get("id") or "", + spotify_artist_ids=list(d.get("artist_ids") or []), + album_image_url=d.get("album_image_url") or "", + release_date=d.get("release_date") or "", + release_date_precision=d.get("release_date_precision") or "", + genres=list(d.get("genres") or []), + track_number=int(d.get("track_number") or 0), + disc_number=int(d.get("disc_number") or 0), + duration_ms=int(d.get("duration_ms") or 0), + source="spotify", + ) + + @classmethod + def from_file(cls, path: Path) -> "Track": + """Construct from on-disk audio via mutagen. Fields not in the file's + tags stay at their defaults; the caller can `merge_spotify` later to + fill them in canonically.""" + from mutagen import File as MutagenFile + + out = cls(path=path, source="local") + try: + audio = MutagenFile(path, easy=True) + except Exception: + return out + if audio is None: + return out + tags = audio.tags or {} + def first(k: str) -> str: + v = tags.get(k) if hasattr(tags, "get") else None + return (v[0] if v else "") if isinstance(v, list) else (v or "") + out.title = first("title") + artist = first("artist") + if artist: + out.artists = [a.strip() for a in artist.split(",") if a.strip()] + out.album = first("album") + out.isrc = first("isrc").strip() + out.release_date = first("date") + gtag = first("genre") + if gtag: + from maple.cli import split_genre_tags + out.genres = split_genre_tags(gtag) + try: + out.bpm = float(first("bpm")) if first("bpm") else None + except ValueError: + out.bpm = None + try: + out.track_number = int(first("tracknumber").split("/")[0]) + except ValueError: + out.track_number = 0 + info = getattr(audio, "info", None) + if info is not None: + out.duration_ms = int(round((getattr(info, "length", 0.0) or 0.0) * 1000)) + bps = int(getattr(info, "bitrate", 0) or 0) + out.bitrate = bps // 1000 if bps > 1000 else bps + out.sample_rate = int(getattr(info, "sample_rate", 0) or 0) + out.sample_depth = int(getattr(info, "bits_per_sample", 0) or 0) + try: + out.file_size = path.stat().st_size + except OSError: + pass + return out + + # ---- enrichment ------------------------------------------------------ # + + def merge_spotify(self, d: dict) -> "Track": + """Overlay Spotify-canonical fields onto this Track *in place* — used + when we have a local file but Spotify carries better identity / genre + / release date. Empty Spotify fields are NOT applied (no clobbering).""" + if d.get("title"): + self.title = d["title"] + if d.get("artists"): + self.artists = list(d["artists"]) + if d.get("album"): + self.album = d["album"] + if d.get("isrc"): + self.isrc = d["isrc"].strip() + if d.get("id"): + self.spotify_track_id = d["id"] + if d.get("artist_ids"): + self.spotify_artist_ids = list(d["artist_ids"]) + if d.get("album_image_url"): + self.album_image_url = d["album_image_url"] + if d.get("release_date"): + self.release_date = d["release_date"] + if d.get("release_date_precision"): + self.release_date_precision = d["release_date_precision"] + if d.get("genres"): + self.genres = list(d["genres"]) + if d.get("duration_ms"): + self.duration_ms = int(d["duration_ms"]) + if d.get("track_number"): + self.track_number = int(d["track_number"]) + if d.get("disc_number"): + self.disc_number = int(d["disc_number"]) + if not self.source: + self.source = "spotify" + return self + + def merge_analysis(self, analysis: dict) -> "Track": + """Merge fields from `maple.analyze.analyze_audio`'s output.""" + if analysis.get("bpm") is not None: + self.bpm = float(analysis["bpm"]) + return self + + # ---- export adapter -------------------------------------------------- # + + def to_pdb_row( + self, + *, + track_id: int, + artist_id: int, + album_id: int, + genre_id: int = 0, + artwork_id: int = 0, + file_path: str, + analyze_path: str = "", + date_added: str = "", + tempo_x100: int | None = None, + ) -> model.Track: + """Project this canonical Track into the denormalised pdb row shape. + + FK ids (`artist_id`, `album_id`, …) and the per-export `file_path` / + `analyze_path` are caller-supplied because they only exist in the + context of a specific export. `tempo_x100` defaults to `round(bpm * + 100)` when we have a BPM; pass through otherwise. + """ + if tempo_x100 is None: + tempo_x100 = round(self.bpm * 100) if self.bpm else 0 + return model.Track( + id=track_id, + title=self.title, + artist_id=artist_id, + album_id=album_id, + filename=Path(file_path).name, + file_path=file_path, + file_size=self.file_size, + sample_rate=self.sample_rate or 44100, + sample_depth=self.sample_depth or 16, + bitrate=self.bitrate, + duration=self.duration_s, + track_number=self.track_number, + year=self.year, + disc_number=self.disc_number, + tempo=tempo_x100, + release_date=self.release_date, + date_added=date_added, + analyze_path=analyze_path, + analyze_date=date_added, + artwork_id=artwork_id, + genre_id=genre_id, + source_path=self.path, + ) diff --git a/maple-py/src/maple/validate.py b/maple-py/src/maple/validate.py new file mode 100644 index 0000000..70a3df7 --- /dev/null +++ b/maple-py/src/maple/validate.py @@ -0,0 +1,162 @@ +"""Cross-cutting per-track validator. + +A track is "valid" iff every export criterion holds. Both `just collection` +(`Collection.process_track`) and `just export` (`cli.build`) must invoke +this — there is no separate "collection-only" or "export-only" validation +in maple. Add new criteria in one place; both pipelines pick them up. + +Criteria (extend as they emerge): + - ffprobe-decodable + - duration > 0 + - title / artist / album / date / genre / isrc / bpm tags present + - embedded cover art present + - analytical BPM cached or fresh (cache key = ISRC) + - if `spotify_track` supplied: tags within fuzzy match of Spotify truth + +`validate_track` returns a list of `TrackIssue`. Empty list = passing. +Callers decide whether to warn-and-continue (the default) or fail. +""" +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from loguru import logger +from mutagen import File as MutagenFile + +from maple.analyze import cache_path_for +from maple.collection.index import ( + first_artist, + fuzzy_ratio, + norm, + norm_loose, +) +from maple.collection.validate import ( + DURATION_TOLERANCE_S, + ffprobe_ok, + file_duration, + has_embedded_art, +) + + +@dataclass +class TrackIssue: + """A single validation finding. + + Shape is (playlist, artist, title, reason) — `reason` is a + `": "` string so structured fields and free-text + descriptions share one container. + """ + + playlist: str + artist: str + title: str + reason: str + + +REQUIRED_TAGS = ("title", "artist", "album", "date", "genre", "bpm", "isrc") + + +def _file_tags(path: Path) -> dict[str, str] | None: + try: + audio = MutagenFile(path, easy=True) + except Exception as e: + logger.warning("could not open {} for validation: {}", path.name, e) + return None + if audio is None or audio.tags is None: + return {k: "" for k in REQUIRED_TAGS} + tags = audio.tags + return {k: (tags.get(k) or [""])[0] if hasattr(tags, "get") else "" for k in REQUIRED_TAGS} + + +def validate_track( + path: Path, + *, + spotify_track: dict | None = None, + analysis_cache_dir: Path | None = None, + playlist: str = "", +) -> list[TrackIssue]: + """Run every criterion against `path`. Returns the list of failed checks. + + Optional context broadens the checks: + - `spotify_track`: enables fuzzy tag comparison against Spotify truth. + - `analysis_cache_dir`: enables the "analysis cached" check via ISRC. + - `playlist`: tags every issue with playlist context (collection flow). + """ + tags = _file_tags(path) + if tags is None: + return [TrackIssue(playlist, "", path.name, "tags: could not read file")] + + artist = tags.get("artist") or ( + ", ".join(spotify_track.get("artists") or []) if spotify_track else "" + ) + title = tags.get("title") or (spotify_track.get("title") if spotify_track else "") or path.stem + issues: list[TrackIssue] = [] + + def fail(field: str, detail: str) -> None: + issues.append(TrackIssue(playlist, artist, title, f"{field}: {detail}")) + + # ---- audio integrity --------------------------------------------------- # + if not ffprobe_ok(path): + fail("decode", f"ffprobe rejected {path.name}") + duration = file_duration(path) + if duration <= 0: + fail("duration", "≤ 0s — corrupt file or unreadable container") + + # ---- required tags ----------------------------------------------------- # + for key in REQUIRED_TAGS: + if not (tags.get(key) or "").strip(): + fail(key, "tag missing") + + # ---- artwork ----------------------------------------------------------- # + if not has_embedded_art(path): + fail("artwork", "no embedded cover art") + + # ---- analysis cached --------------------------------------------------- # + isrc = (tags.get("isrc") or "").strip() + if analysis_cache_dir is not None and isrc: + cp = cache_path_for(analysis_cache_dir, isrc) + if cp is None or not cp.exists(): + fail("analysis", f"no cached analysis at {cp}") + + # ---- Spotify reconciliation -------------------------------------------- # + if spotify_track is not None: + sp_artist = ", ".join(spotify_track.get("artists") or []) + sp_title = (spotify_track.get("title") or "").strip() + sp_album = (spotify_track.get("album") or "").strip() + sp_isrc = (spotify_track.get("isrc") or "").strip() + sp_duration = (spotify_track.get("duration_ms") or 0) / 1000.0 + sp_date = (spotify_track.get("release_date") or "").strip() + + f_artist = tags.get("artist") or "" + f_title = tags.get("title") or "" + f_album = tags.get("album") or "" + f_isrc = tags.get("isrc") or "" + f_date = tags.get("date") or "" + + if sp_artist and fuzzy_ratio(norm(first_artist(sp_artist)), norm(first_artist(f_artist))) < 0.5: + fail("artist", f"spotify={sp_artist!r} file={f_artist!r}") + if sp_title and fuzzy_ratio(norm_loose(sp_title), norm_loose(f_title)) < 0.5: + fail("title", f"spotify={sp_title!r} file={f_title!r}") + if sp_album and f_album and fuzzy_ratio(norm_loose(sp_album), norm_loose(f_album)) < 0.5: + fail("album", f"spotify={sp_album!r} file={f_album!r}") + elif sp_album and not f_album: + fail("album", f"spotify={sp_album!r} file=''") + if sp_isrc and f_isrc and sp_isrc.upper() != f_isrc.upper(): + fail("isrc", f"spotify={sp_isrc!r} file={f_isrc!r}") + if sp_date and f_date and sp_date[:4] != f_date[:4]: + fail("date", f"spotify={sp_date!r} file={f_date!r}") + if sp_duration > 0 and abs(duration - sp_duration) > DURATION_TOLERANCE_S: + fail("duration", f"file {duration:.1f}s vs spotify {sp_duration:.1f}s") + + return issues + + +def log_issues(issues: list[TrackIssue], *, report: bool = True) -> None: + """Emit every issue via loguru.warning. When `report=True`, also bind + `extra["report"]` so any attached file-sink (Collection.attach_report_sink + or the export-side equivalent) captures the line.""" + bound = logger.bind(report=True) if report else logger + for i in issues: + prefix = f"[{i.playlist}] " if i.playlist else "" + bound.warning("{}{} - {}: {}", prefix, i.artist, i.title, i.reason) diff --git a/maple-py/uv.lock b/maple-py/uv.lock index a23b16b..ed653b8 100644 --- a/maple-py/uv.lock +++ b/maple-py/uv.lock @@ -90,6 +90,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "aubio" +version = "0.4.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/80/302d89240603e5347c7f8026c8b02c59f8dfaec66c91a743d82de7c86006/aubio-0.4.9.tar.gz", hash = "sha256:df1244f6c4cf5bea382c8c2d35aa43bc31f4cf631fe325ae3992c219546a4202", size = 479008, upload-time = "2019-02-08T11:21:02.438Z" } + [[package]] name = "bidict" version = "0.23.1" @@ -243,10 +252,12 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, + { name = "aubio" }, { name = "kaitaistruct" }, { name = "loguru" }, { name = "mutagen" }, { name = "pillow" }, + { name = "polars" }, { name = "pyrekordbox" }, ] @@ -258,10 +269,12 @@ dev = [ [package.metadata] requires-dist = [ { name = "aiohttp", specifier = ">=3.10" }, + { name = "aubio", specifier = ">=0.4.9" }, { name = "kaitaistruct", specifier = ">=0.11" }, { name = "loguru", specifier = ">=0.7" }, { name = "mutagen", specifier = ">=1.47.0" }, { name = "pillow", specifier = ">=12.2.0" }, + { name = "polars", specifier = ">=1.0" }, { name = "pyrekordbox", specifier = ">=0.4.4" }, ] @@ -402,6 +415,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "polars" +version = "1.41.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "polars-runtime-32" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/f9/aeda46259b0669247a160315d2d51269de9504b9dd2f70acadbcb22f46b7/polars-1.41.2.tar.gz", hash = "sha256:256d6731162371b77f3f29a55eacb8c0fc740ddb1a293a01d2ef5b5393c5c708", size = 737996, upload-time = "2026-05-29T17:39:15.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/22/28f62d24f7db56ac4343588f9362d49b7b4177e55ac47a466fe696b0099b/polars-1.41.2-py3-none-any.whl", hash = "sha256:23ce9a2910b6e3e8d4258770bf44aa17170958df7af6e85feedf4458a04d8d29", size = 833445, upload-time = "2026-05-29T17:37:05.576Z" }, +] + +[[package]] +name = "polars-runtime-32" +version = "1.41.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/56/54e3ea0e9b64f327179049e4742241cc6b1d3e8fa414b05a057dd26df367/polars_runtime_32-1.41.2.tar.gz", hash = "sha256:7af09ec1ab053da2c9669e8d15f809a4083a29be05db57111688b8051062af56", size = 2989474, upload-time = "2026-05-29T17:39:17.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/9b/fe72a3811c0357cdb06c67bdc7695fa1623ad47948fc523195f5ac31037f/polars_runtime_32-1.41.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:95a08346dac337357cdb825c8076df7d36da54c4caa59a5cb41d0a30691c5edd", size = 52265283, upload-time = "2026-05-29T17:37:09.407Z" }, + { url = "https://files.pythonhosted.org/packages/0a/93/fab9da803fd80d9e83ef88c20932f637a10bc611b20415fc322eec84bc44/polars_runtime_32-1.41.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:dedfaeec2c7f995298da7319dd9431d662e5dd1d0ec51b1459df4a0234ceff52", size = 46571222, upload-time = "2026-05-29T17:37:13.698Z" }, + { url = "https://files.pythonhosted.org/packages/c8/2a/8843f34a8ac57acd058a39b87b03b580dd352a490e9dae0415e02033bdd4/polars_runtime_32-1.41.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18eea22c5cc34e27f8a60950458ad81e6a9ea75e89363ca1367e14e7e7f781fc", size = 50409372, upload-time = "2026-05-29T17:37:17.875Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c6/92b352fe88cf51bd0a19fb99e1c0cbe46aa26c14dcf7995b89869cd932ae/polars_runtime_32-1.41.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2630540dfdfb0f36f9b04a07c7c2e3f50bf2ad384113263c1c812007ee9141e0", size = 56405484, upload-time = "2026-05-29T17:37:22.684Z" }, + { url = "https://files.pythonhosted.org/packages/74/c4/bae3174c3b02f6b441d2e58594387abcd509f67a098f682a83b195f08966/polars_runtime_32-1.41.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:20e969e08f9b137e233c04cc04de73d9795f89eb77d34854e40a025965a43763", size = 50603512, upload-time = "2026-05-29T17:37:27.422Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ed/f2d26ae02d92c2689056838ed59e2a626326ad23c2831d58637d25f6c82a/polars_runtime_32-1.41.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e7016a3deb641b64a31447abbbee0f34bd020a6a9ae34ee6b743837def15e2a4", size = 54328561, upload-time = "2026-05-29T17:37:32.587Z" }, + { url = "https://files.pythonhosted.org/packages/9b/c4/9c3831cc885dc7769e59abf8f583821a5fb4403fd0e4eba0ccc6d47a3d4b/polars_runtime_32-1.41.2-cp310-abi3-win_amd64.whl", hash = "sha256:1e5e5377c315e0dcafdfb2a31adc546abbaeb3f9cb1864e6536523d2af473265", size = 51978643, upload-time = "2026-05-29T17:37:37.443Z" }, + { url = "https://files.pythonhosted.org/packages/cd/c6/79e9f3f270270d7ed5575d92b7bfef49f01abd9275447161275b23b553a8/polars_runtime_32-1.41.2-cp310-abi3-win_arm64.whl", hash = "sha256:843d96f69d18eca53429c1198e58891db7f18111f83b9c419bb45ad9d73eaed5", size = 46006901, upload-time = "2026-05-29T17:37:42.522Z" }, +] + [[package]] name = "propcache" version = "0.5.2" diff --git a/maple-ts/README.md b/maple-ts/README.md deleted file mode 100644 index 5dfa598..0000000 --- a/maple-ts/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# maple-ts - -TypeScript/Bun port of `maple-py`. Builds a rekordbox `export.pdb` USB layout from a folder of audio files and `.m3u8` playlists. - -## Install - -```sh -bun install -``` - -## Run - -```sh -bun run src/cli.ts ./data/input ./data/usb -``` - -Requires `ffmpeg` on `PATH` for FLAC -> MP3 conversion. diff --git a/maple-ts/package.json b/maple-ts/package.json deleted file mode 100644 index d039581..0000000 --- a/maple-ts/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "maple", - "version": "0.1.0", - "description": "Build a rekordbox USB from a folder of audio + m3u8 playlists", - "type": "module", - "module": "src/cli.ts", - "bin": { - "maple": "src/cli.ts" - }, - "scripts": { - "maple": "bun run src/cli.ts", - "test": "bun test" - }, - "devDependencies": { - "@types/bun": "latest", - "@types/node-id3": "^0.2.6" - }, - "dependencies": { - "music-metadata": "^10.5.0", - "node-id3": "^0.2.6" - } -} diff --git a/maple-ts/src/cli.ts b/maple-ts/src/cli.ts deleted file mode 100644 index 9695705..0000000 --- a/maple-ts/src/cli.ts +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env bun -/** maple CLI: scan a directory of audio + m3u8 playlists and build a rekordbox USB. */ -import { stat } from "node:fs/promises"; -import { basename, extname, join, resolve } from "node:path"; -import os from "node:os"; - -import { convertFlacToMp3, verifyDuration } from "./convert.ts"; -import { stageAudio, stagedPath } from "./copy.ts"; -import { readTags } from "./meta.ts"; -import { newDatabase, newTrack, type Database } from "./model.ts"; -import { writePdb } from "./pdb/writer.ts"; -import { scanAudio, scanPlaylists } from "./scan.ts"; - -function today(): string { - return new Date().toISOString().slice(0, 10); -} - -function withSuffix(file: string, newExt: string): string { - const ext = extname(file); - return file.slice(0, file.length - ext.length) + newExt; -} - -async function isDir(p: string): Promise { - try { return (await stat(p)).isDirectory(); } catch { return false; } -} - -async function pMap(items: T[], workers: number, fn: (item: T) => Promise): Promise { - const out: R[] = new Array(items.length); - let i = 0; - const runners = Array.from({ length: Math.max(1, Math.min(workers, items.length)) }, async () => { - while (true) { - const idx = i++; - if (idx >= items.length) return; - out[idx] = await fn(items[idx]!); - } - }); - await Promise.all(runners); - return out; -} - -export async function build(inputDir: string, usbRoot: string): Promise { - const db = newDatabase(); - const dateAdded = today(); - - const artistIndex = new Map(); - const albumIndex = new Map(); - const trackBySrc = new Map(); - - const audioFiles = await scanAudio(inputDir); - - const initialMeta = new Map>>(); - for (const p of audioFiles) initialMeta.set(p, await readTags(p)); - - const flacJobs: Array<{ src: string; dst: string }> = []; - for (const original of audioFiles) { - if (extname(original).toLowerCase() === ".flac") { - const m = initialMeta.get(original)!; - const mp3Name = withSuffix(basename(original), ".mp3"); - const { destAbs } = stagedPath(usbRoot, m.artist, m.album, mp3Name); - flacJobs.push({ src: original, dst: destAbs }); - } - } - - const workers = Math.max(1, os.cpus().length || 1); - const results = await pMap(flacJobs, workers, async (job) => { - const converted = await convertFlacToMp3(job.src, job.dst); - return { ...job, converted }; - }); - for (const r of results) { - const rel = r.src.startsWith(inputDir) ? r.src.slice(inputDir.length).replace(/^[\/\\]+/, "") : r.src; - console.log(`${r.converted ? "convert" : "cached "} ${rel}`); - await verifyDuration(r.src, r.dst); - } - - for (const original of audioFiles) { - let meta = initialMeta.get(original)!; - let artistId = artistIndex.get(meta.artist); - if (artistId === undefined) { - artistId = db.artists.length + 1; - artistIndex.set(meta.artist, artistId); - db.artists.push({ id: artistId, name: meta.artist }); - } - const albumKey = `${artistId}\x00${meta.album}`; - let albumId = albumIndex.get(albumKey); - if (albumId === undefined) { - albumId = db.albums.length + 1; - albumIndex.set(albumKey, albumId); - db.albums.push({ id: albumId, name: meta.album, artist_id: artistId }); - } - - let filePath: string; - if (extname(original).toLowerCase() === ".flac") { - const mp3Name = withSuffix(basename(original), ".mp3"); - const s = stagedPath(usbRoot, meta.artist, meta.album, mp3Name); - filePath = s.filePath; - meta = await readTags(s.destAbs); - } else { - const s = await stageAudio(original, usbRoot, meta.artist, meta.album); - filePath = s.filePath; - } - - const trackId = db.tracks.length + 1; - db.tracks.push(newTrack({ - id: trackId, - title: meta.title, - artist_id: artistId, - album_id: albumId, - filename: basename(filePath), - file_path: filePath, - file_size: meta.file_size, - sample_rate: meta.sample_rate, - sample_depth: meta.sample_depth, - bitrate: meta.bitrate, - duration: meta.duration, - track_number: meta.track_number, - year: meta.year, - date_added: dateAdded, - source_path: original, - })); - trackBySrc.set(resolve(original), trackId); - } - - const playlists = await scanPlaylists(inputDir); - let sortOrder = 1; - for (const [name, paths] of playlists) { - const playlistId = db.playlist_nodes.length + 1; - db.playlist_nodes.push({ - id: playlistId, - parent_id: 0, - sort_order: sortOrder, - name, - is_folder: false, - }); - sortOrder += 1; - let entryIndex = 1; - for (const p of paths) { - const tid = trackBySrc.get(resolve(p)); - if (tid === undefined) { - console.error(`warning: playlist '${name}' references missing file ${p}`); - continue; - } - db.playlist_entries.push({ - playlist_id: playlistId, - track_id: tid, - entry_index: entryIndex, - }); - entryIndex += 1; - } - } - - return db; -} - -async function main(argv: string[]): Promise { - const args = argv.slice(2); - const inputDir = args[0] ?? "./data/input"; - const outputRoot = args[1] ?? "./data/usb"; - - if (!(await isDir(inputDir))) { - console.error(`error: input_dir ${inputDir} is not a directory`); - return 2; - } - - const db = await build(inputDir, outputRoot); - const pdbPath = join(outputRoot, "PIONEER", "rekordbox", "export.pdb"); - await writePdb(db, pdbPath); - - console.log( - `wrote ${db.tracks.length} tracks, ${db.artists.length} artists, ${db.albums.length} albums, ` + - `${db.playlist_nodes.length} playlists (${db.playlist_entries.length} entries) to ${pdbPath}` - ); - return 0; -} - -if (import.meta.main) { - process.exit(await main(process.argv)); -} diff --git a/maple-ts/src/convert.ts b/maple-ts/src/convert.ts deleted file mode 100644 index ea67673..0000000 --- a/maple-ts/src/convert.ts +++ /dev/null @@ -1,90 +0,0 @@ -/** - * FLAC -> MP3 V0 joint-stereo conversion via ffmpeg, with source-hash dedup. - * - * The SHA-256 of the source FLAC is stamped into a TXXX:MAPLE_SRC_SHA256 ID3 - * frame on the produced MP3. Reruns skip conversion when the existing MP3 - * carries the matching hash, so the cache is content-addressed against inputs. - */ -import { mkdir, access } from "node:fs/promises"; -import { dirname } from "node:path"; -import { parseFile } from "music-metadata"; -import NodeID3 from "node-id3"; - -const HASH_FRAME_DESC = "MAPLE_SRC_SHA256"; - -export async function fileSha256(path: string): Promise { - const hasher = new Bun.CryptoHasher("sha256"); - const stream = Bun.file(path).stream(); - for await (const chunk of stream) hasher.update(chunk); - return hasher.digest("hex"); -} - -async function exists(p: string): Promise { - try { await access(p); return true; } catch { return false; } -} - -function readSourceHash(mp3: string): string | null { - const tags = NodeID3.read(mp3); - const frames = tags.userDefinedText; - if (!frames) return null; - for (const f of frames) { - if (f.description === HASH_FRAME_DESC && f.value) return f.value; - } - return null; -} - -function writeSourceHash(mp3: string, sha: string): void { - NodeID3.update({ userDefinedText: [{ description: HASH_FRAME_DESC, value: sha }] }, mp3); -} - -async function which(cmd: string): Promise { - const p = Bun.spawn({ cmd: ["which", cmd], stdout: "pipe", stderr: "pipe" }); - await p.exited; - return p.exitCode === 0; -} - -export async function convertFlacToMp3(flac: string, mp3: string): Promise { - const srcHash = await fileSha256(flac); - if (await exists(mp3)) { - try { - if (readSourceHash(mp3) === srcHash) return false; - } catch {} - } - - if (!(await which("ffmpeg"))) throw new Error("ffmpeg not found on PATH"); - - await mkdir(dirname(mp3), { recursive: true }); - const proc = Bun.spawn({ - cmd: [ - "ffmpeg", "-nostdin", "-loglevel", "error", "-y", - "-i", flac, - "-codec:a", "libmp3lame", - "-q:a", "0", - "-joint_stereo", "1", - "-map_metadata", "0", - mp3, - ], - stdout: "inherit", - stderr: "inherit", - }); - await proc.exited; - if (proc.exitCode !== 0) throw new Error(`ffmpeg failed for ${flac}`); - writeSourceHash(mp3, srcHash); - return true; -} - -async function duration(path: string): Promise { - try { - const md = await parseFile(path, { duration: true }); - return md.format.duration ?? 0; - } catch { - return 0; - } -} - -export async function verifyDuration(src: string, dst: string, tolerance = 0.1): Promise { - const [a, b] = await Promise.all([duration(src), duration(dst)]); - if (Math.abs(a - b) > tolerance) { - throw new Error(`duration mismatch: ${src} = ${a.toFixed(3)}s vs ${dst} = ${b.toFixed(3)}s (Δ ${Math.abs(a - b).toFixed(3)}s)`); - } -} diff --git a/maple-ts/src/copy.ts b/maple-ts/src/copy.ts deleted file mode 100644 index 7b6b42a..0000000 --- a/maple-ts/src/copy.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { dirname, join, basename } from "node:path"; -import { mkdir, copyFile, access } from "node:fs/promises"; - -const BAD = /[\\/:*?"<>|\x00-\x1f]/g; - -export function sanitise(component: string): string { - const s = component.replace(BAD, "_").replace(/^[ .]+|[ .]+$/g, ""); - return s || "_"; -} - -export function stagedPath(usbRoot: string, artist: string, album: string, filename: string): { destAbs: string; filePath: string } { - const relParts = ["Contents", sanitise(artist), sanitise(album), sanitise(filename)]; - const destAbs = join(usbRoot, ...relParts); - const filePath = "/" + relParts.join("/"); - return { destAbs, filePath }; -} - -async function exists(p: string): Promise { - try { await access(p); return true; } catch { return false; } -} - -export async function stageAudio(src: string, usbRoot: string, artist: string, album: string): Promise<{ destAbs: string; filePath: string }> { - const { destAbs, filePath } = stagedPath(usbRoot, artist, album, basename(src)); - await mkdir(dirname(destAbs), { recursive: true }); - if (!(await exists(destAbs))) { - await copyFile(src, destAbs); - } - return { destAbs, filePath }; -} diff --git a/maple-ts/src/meta.ts b/maple-ts/src/meta.ts deleted file mode 100644 index a0ab917..0000000 --- a/maple-ts/src/meta.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { parseFile } from "music-metadata"; -import { basename, extname } from "node:path"; -import { stat } from "node:fs/promises"; - -export interface TrackMeta { - title: string; - artist: string; - album: string; - track_number: number; - year: number; - duration: number; - bitrate: number; - sample_rate: number; - sample_depth: number; - file_size: number; -} - -function toInt(v: unknown): number { - if (v === null || v === undefined) return 0; - let s = String(Array.isArray(v) ? v[0] : v).trim(); - if (!s) return 0; - s = s.split("/")[0]!; - const n = parseInt(s, 10); - return Number.isFinite(n) ? n : 0; -} - -export async function readTags(path: string): Promise { - const { size: file_size } = await stat(path); - const stem = basename(path, extname(path)); - try { - const md = await parseFile(path, { duration: true }); - const c = md.common; - const info = md.format; - const bitrate_bps = info.bitrate ?? 0; - const bitrate = bitrate_bps > 1000 ? Math.floor(bitrate_bps / 1000) : bitrate_bps; - return { - title: c.title || stem, - artist: c.artist || "Unknown Artist", - album: c.album || "Unknown Album", - track_number: toInt(c.track?.no), - year: toInt(c.year ?? c.date), - duration: Math.floor(info.duration ?? 0), - bitrate, - sample_rate: info.sampleRate ?? 44100, - sample_depth: info.bitsPerSample ?? 16, - file_size, - }; - } catch { - return { - title: stem, - artist: "Unknown Artist", - album: "Unknown Album", - track_number: 0, - year: 0, - duration: 0, - bitrate: 0, - sample_rate: 44100, - sample_depth: 16, - file_size, - }; - } -} diff --git a/maple-ts/src/model.ts b/maple-ts/src/model.ts deleted file mode 100644 index 9edc478..0000000 --- a/maple-ts/src/model.ts +++ /dev/null @@ -1,83 +0,0 @@ -export interface Artist { - id: number; - name: string; -} - -export interface Album { - id: number; - name: string; - artist_id: number; -} - -export interface Track { - id: number; - title: string; - artist_id: number; - album_id: number; - filename: string; - file_path: string; - file_size: number; - sample_rate: number; - sample_depth: number; - bitrate: number; - duration: number; - track_number: number; - year: number; - disc_number: number; - tempo: number; - comment: string; - date_added: string; - release_date: string; - mix_name: string; - analyze_path: string; - analyze_date: string; - source_path: string | null; -} - -export interface PlaylistNode { - id: number; - parent_id: number; - sort_order: number; - name: string; - is_folder: boolean; -} - -export interface PlaylistEntry { - playlist_id: number; - track_id: number; - entry_index: number; -} - -export interface Database { - tracks: Track[]; - artists: Artist[]; - albums: Album[]; - playlist_nodes: PlaylistNode[]; - playlist_entries: PlaylistEntry[]; -} - -export function newDatabase(): Database { - return { tracks: [], artists: [], albums: [], playlist_nodes: [], playlist_entries: [] }; -} - -export function newTrack(init: Pick & Partial): Track { - return { - file_size: 0, - sample_rate: 44100, - sample_depth: 16, - bitrate: 0, - duration: 0, - track_number: 0, - year: 0, - disc_number: 0, - tempo: 0, - comment: "", - date_added: "", - release_date: "", - mix_name: "", - analyze_path: "", - analyze_date: "", - source_path: null, - ...init, - }; -} diff --git a/maple-ts/src/pdb/buf.ts b/maple-ts/src/pdb/buf.ts deleted file mode 100644 index 0de201c..0000000 --- a/maple-ts/src/pdb/buf.ts +++ /dev/null @@ -1,44 +0,0 @@ -/** Small little-endian write helpers backed by Uint8Array. */ -export class LE { - parts: Uint8Array[] = []; - private len = 0; - - push(u: Uint8Array): void { this.parts.push(u); this.len += u.length; } - - u8(v: number): void { - const b = new Uint8Array(1); b[0] = v & 0xff; this.push(b); - } - u16(v: number): void { - const b = new Uint8Array(2); - b[0] = v & 0xff; - b[1] = (v >> 8) & 0xff; - this.push(b); - } - u32(v: number): void { - const b = new Uint8Array(4); - b[0] = v & 0xff; - b[1] = (v >>> 8) & 0xff; - b[2] = (v >>> 16) & 0xff; - b[3] = (v >>> 24) & 0xff; - this.push(b); - } - bytes(b: Uint8Array): void { this.push(b); } - - length(): number { return this.len; } - - build(): Uint8Array { - const out = new Uint8Array(this.len); - let o = 0; - for (const p of this.parts) { out.set(p, o); o += p.length; } - return out; - } -} - -export function concat(parts: Uint8Array[]): Uint8Array { - let len = 0; - for (const p of parts) len += p.length; - const out = new Uint8Array(len); - let o = 0; - for (const p of parts) { out.set(p, o); o += p.length; } - return out; -} diff --git a/maple-ts/src/pdb/pager.ts b/maple-ts/src/pdb/pager.ts deleted file mode 100644 index 58b083c..0000000 --- a/maple-ts/src/pdb/pager.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { LE } from "./buf.ts"; - -export const PAGE_SIZE = 4096; -const HEADER_SIZE = 40; -const ROW_GROUP_SIZE = 36; -const ROWS_PER_GROUP = 16; - -function align4(o: number): number { return (o + 3) & ~3; } -function numRowGroups(n: number): number { return Math.floor((n + ROWS_PER_GROUP - 1) / ROWS_PER_GROUP); } - -export class PageBuilder { - pageIndex: number; - pageType: number; - nextPage: number; - rowBytes: Uint8Array[] = []; - rowOffsets: number[] = []; - heapCursor = 0; - - constructor(pageIndex: number, pageType: number) { - this.pageIndex = pageIndex; - this.pageType = pageType; - this.nextPage = pageIndex; - } - - tryAdd(row: Uint8Array): boolean { - const aligned = align4(this.heapCursor); - const end = aligned + row.length; - const footerAfter = numRowGroups(this.rowBytes.length + 1) * ROW_GROUP_SIZE; - if (end + HEADER_SIZE + footerAfter > PAGE_SIZE) return false; - this.rowOffsets.push(aligned); - this.rowBytes.push(row); - this.heapCursor = end; - return true; - } - - numRows(): number { return this.rowBytes.length; } - - finalize(): Uint8Array { - const n = this.rowBytes.length; - const numRgs = numRowGroups(n); - const footerSize = numRgs * ROW_GROUP_SIZE; - const heapPadding = PAGE_SIZE - HEADER_SIZE - footerSize; - const usedSize = this.heapCursor; - const freeSize = heapPadding - usedSize; - if (freeSize < 0) throw new Error("page overflow"); - - const pageFlags = n > 0 ? 0x00 : 0x44; - - const h = new LE(); - h.u32(0); // magic - h.u32(this.pageIndex); - h.u32(this.pageType); - h.u32(this.nextPage); - h.u32(0); // unknown1 - h.u32(0); // unknown2 - h.u8(n & 0xff); h.u8(0); h.u8(0); h.u8(pageFlags); - h.u16(freeSize); h.u16(usedSize); - h.u16(0); // unknown5 - h.u16(n); // num_rows_large - h.u16(0); // unknown6 - h.u16(0); // unknown7 - const header = h.build(); - if (header.length !== HEADER_SIZE) throw new Error(`hdr len ${header.length}`); - - const heap = new Uint8Array(heapPadding); - for (let i = 0; i < this.rowBytes.length; i++) { - heap.set(this.rowBytes[i]!, this.rowOffsets[i]!); - } - - const footer = new LE(); - for (let g = numRgs - 1; g >= 0; g--) { - const start = g * ROWS_PER_GROUP; - const grpOffsets = this.rowOffsets.slice(start, start + ROWS_PER_GROUP); - const grpCount = grpOffsets.length; - const padded = grpOffsets.slice(); - while (padded.length < ROWS_PER_GROUP) padded.push(0); - const presence = (1 << grpCount) - 1; - const rg = new LE(); - for (let i = padded.length - 1; i >= 0; i--) rg.u16(padded[i]!); - rg.u16(presence); - rg.u16(16); - const rgBytes = rg.build(); - if (rgBytes.length !== ROW_GROUP_SIZE) throw new Error("rg size"); - footer.bytes(rgBytes); - } - const footerBytes = footer.build(); - if (footerBytes.length !== footerSize) throw new Error("footer size"); - - const out = new Uint8Array(PAGE_SIZE); - out.set(header, 0); - out.set(heap, HEADER_SIZE); - out.set(footerBytes, HEADER_SIZE + heapPadding); - return out; - } -} diff --git a/maple-ts/src/pdb/rows.ts b/maple-ts/src/pdb/rows.ts deleted file mode 100644 index 0a15d0d..0000000 --- a/maple-ts/src/pdb/rows.ts +++ /dev/null @@ -1,148 +0,0 @@ -import type { Album, Artist, PlaylistEntry, PlaylistNode, Track } from "../model.ts"; -import { encodeDevicesql } from "./strings.ts"; -import { LE, concat } from "./buf.ts"; - -const TRACK_FIXED_PREFIX = 0x5e; -const TRACK_OFS_STRINGS_START = TRACK_FIXED_PREFIX; -const TRACK_HEADER_LEN = TRACK_OFS_STRINGS_START + 21 * 2; - -function pad2(b: Uint8Array): Uint8Array { - if (b.length & 1) { - const out = new Uint8Array(b.length + 1); - out.set(b); - return out; - } - return b; -} - -export function serializeTrack(t: Track): Uint8Array { - const strings: string[] = [ - "", // 0 isrc - "", // 1 texter - "1", // 2 unknown_string_2 - "2", // 3 unknown_string_3 - "", // 4 unknown_string_4 - "", // 5 message - "ON", // 6 kuvo_public - "ON", // 7 autoload_hot_cues - "", // 8 unknown_string_5 - "", // 9 unknown_string_6 - t.date_added, // 10 - t.release_date, // 11 - t.mix_name, // 12 - "", // 13 - t.analyze_path, // 14 - t.analyze_date, // 15 - t.comment, // 16 - t.title, // 17 - "", // 18 - t.filename, // 19 - t.file_path, // 20 - ]; - - const encoded = strings.map(encodeDevicesql); - const offsets: number[] = []; - let cursor = TRACK_HEADER_LEN; - for (const e of encoded) { - offsets.push(cursor); - cursor += e.length; - } - const blob = concat(encoded); - - const p = new LE(); - p.u16(0x24); p.u16(0); // subtype, index_shift - p.u32(788224); // bitmask - p.u32(t.sample_rate); - p.u32(0); // composer_id - p.u32(t.file_size); - p.u32(0); // unknown - p.u16(19048); - p.u16(30967); - p.u32(0); // artwork_id - p.u32(0); // key_id - p.u32(0); // original_artist_id - p.u32(0); // label_id - p.u32(0); // remixer_id - p.u32(t.bitrate); - p.u32(t.track_number); - p.u32(t.tempo); - p.u32(0); // genre_id - p.u32(t.album_id); - p.u32(t.artist_id); - p.u32(t.id); - p.u16(t.disc_number); - p.u16(0); // play_count - p.u16(t.year); - p.u16(t.sample_depth); - p.u16(t.duration); - p.u16(41); - p.u8(0); p.u8(0); // color_id, rating - p.u16(1); - p.u16(3); - - const prefix = p.build(); - if (prefix.length !== TRACK_FIXED_PREFIX) throw new Error(`prefix len ${prefix.length}`); - - const ofs = new LE(); - for (const o of offsets) ofs.u16(o); - return pad2(concat([prefix, ofs.build(), blob])); -} - -export function serializeArtist(a: Artist): Uint8Array { - const name = encodeDevicesql(a.name); - const nearOfs = 0x0a; - const farThreshold = 0x100; - const r = new LE(); - if (nearOfs < farThreshold) { - r.u16(0x60); r.u16(a.id & 0xffff); - r.u32(a.id); - r.u8(0x03); - r.u8(nearOfs); - if (r.length() !== nearOfs) throw new Error("artist near offset mismatch"); - r.bytes(name); - } else { - const far = 0x0e; - r.u16(0x64); r.u16(a.id & 0xffff); - r.u32(a.id); - r.u8(0x03); - r.u8(0); - r.u16(far); - r.bytes(name); - } - return pad2(r.build()); -} - -export function serializeAlbum(a: Album): Uint8Array { - const name = encodeDevicesql(a.name); - const nearOfs = 0x16; - const r = new LE(); - r.u16(0x80); r.u16(a.id & 0xffff); - r.u32(0); - r.u32(a.artist_id); - r.u32(a.id); - r.u32(0); - r.u8(0x03); - r.u8(nearOfs); - if (r.length() !== nearOfs) throw new Error("album near offset mismatch"); - r.bytes(name); - return pad2(r.build()); -} - -export function serializePlaylistTree(n: PlaylistNode): Uint8Array { - const r = new LE(); - r.u32(n.parent_id); - r.u32(0); - r.u32(n.sort_order); - r.u32(n.id); - r.u32(n.is_folder ? 1 : 0); - r.bytes(encodeDevicesql(n.name)); - return pad2(r.build()); -} - -export function serializePlaylistEntry(e: PlaylistEntry): Uint8Array { - const r = new LE(); - r.u32(e.entry_index); - r.u32(e.track_id); - r.u32(e.playlist_id); - return r.build(); -} diff --git a/maple-ts/src/pdb/strings.ts b/maple-ts/src/pdb/strings.ts deleted file mode 100644 index 265476f..0000000 --- a/maple-ts/src/pdb/strings.ts +++ /dev/null @@ -1,33 +0,0 @@ -const EMPTY = new Uint8Array([0x03]); - -function isAscii(s: string): boolean { - for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) > 0x7f) return false; - return true; -} - -export function encodeDevicesql(s: string): Uint8Array { - if (!s) return EMPTY; - if (isAscii(s) && s.length < 127) { - const ascii = new TextEncoder().encode(s); - const length = ascii.length + 1; - const lengthAndKind = ((length << 1) | 1) & 0xff; - const out = new Uint8Array(1 + ascii.length); - out[0] = lengthAndKind; - out.set(ascii, 1); - return out; - } - const utf16 = new Uint8Array(s.length * 2); - for (let i = 0; i < s.length; i++) { - const c = s.charCodeAt(i); - utf16[i * 2] = c & 0xff; - utf16[i * 2 + 1] = (c >> 8) & 0xff; - } - const totalLength = utf16.length + 4; - const out = new Uint8Array(4 + utf16.length); - out[0] = 0x90; - out[1] = totalLength & 0xff; - out[2] = (totalLength >> 8) & 0xff; - out[3] = 0x00; - out.set(utf16, 4); - return out; -} diff --git a/maple-ts/src/pdb/writer.ts b/maple-ts/src/pdb/writer.ts deleted file mode 100644 index 0f066c9..0000000 --- a/maple-ts/src/pdb/writer.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { mkdir, writeFile } from "node:fs/promises"; -import { dirname } from "node:path"; -import type { Database } from "../model.ts"; -import { PAGE_SIZE, PageBuilder } from "./pager.ts"; -import { - serializeAlbum, - serializeArtist, - serializePlaylistEntry, - serializePlaylistTree, - serializeTrack, -} from "./rows.ts"; -import { LE } from "./buf.ts"; - -const PAGE_TYPE_TRACKS = 0; -const PAGE_TYPE_ARTISTS = 2; -const PAGE_TYPE_ALBUMS = 3; -const PAGE_TYPE_PLAYLIST_TREE = 7; -const PAGE_TYPE_PLAYLIST_ENTRIES = 8; - -const FILE_HEADER_SIZE = 28; -const TABLE_ENTRY_SIZE = 16; - -interface Chain { - pageType: number; - pages: Uint8Array[]; - first: number; - last: number; -} - -function buildChain(pageType: number, rows: Uint8Array[], startIndex: number): Chain { - const builders: PageBuilder[] = []; - let idx = startIndex; - let current = new PageBuilder(idx, pageType); - - for (const row of rows) { - if (!current.tryAdd(row)) { - builders.push(current); - idx += 1; - current = new PageBuilder(idx, pageType); - if (!current.tryAdd(row)) { - throw new Error(`row of size ${row.length} too large for an empty page`); - } - } - } - builders.push(current); - - const out: Uint8Array[] = []; - for (let i = 0; i < builders.length; i++) { - const p = builders[i]!; - if (i === builders.length - 1) p.nextPage = p.pageIndex; - else p.nextPage = p.pageIndex + 1; - out.push(p.finalize()); - } - - return { pageType, pages: out, first: startIndex, last: idx }; -} - -export async function writePdb(db: Database, path: string): Promise { - const chains: Chain[] = []; - let nextIdx = 1; - - const tables: Array<[number, Uint8Array[]]> = [ - [PAGE_TYPE_TRACKS, db.tracks.map(serializeTrack)], - [PAGE_TYPE_ARTISTS, db.artists.map(serializeArtist)], - [PAGE_TYPE_ALBUMS, db.albums.map(serializeAlbum)], - [PAGE_TYPE_PLAYLIST_TREE, db.playlist_nodes.map(serializePlaylistTree)], - [PAGE_TYPE_PLAYLIST_ENTRIES, db.playlist_entries.map(serializePlaylistEntry)], - ]; - - for (const [ptype, rows] of tables) { - const c = buildChain(ptype, rows, nextIdx); - chains.push(c); - nextIdx = c.last + 1; - } - - const numTables = chains.length; - const nextUnused = nextIdx; - - const h = new LE(); - h.u32(0); - h.u32(PAGE_SIZE); - h.u32(numTables); - h.u32(nextUnused); - h.u32(0); - h.u32(1); - h.u32(0); - const fileHeader = h.build(); - if (fileHeader.length !== FILE_HEADER_SIZE) throw new Error("file header size"); - - const tbl = new LE(); - for (const c of chains) { - tbl.u32(c.pageType); - tbl.u32(0); - tbl.u32(c.first); - tbl.u32(c.last); - } - const tablesBlock = tbl.build(); - if (tablesBlock.length !== numTables * TABLE_ENTRY_SIZE) throw new Error("tables size"); - - const headerFull = new Uint8Array(fileHeader.length + tablesBlock.length); - headerFull.set(fileHeader, 0); - headerFull.set(tablesBlock, fileHeader.length); - if (headerFull.length > PAGE_SIZE) throw new Error("too many tables for one page"); - - const page0 = new Uint8Array(PAGE_SIZE); - page0.set(headerFull, 0); - - let totalLen = PAGE_SIZE; - for (const c of chains) totalLen += c.pages.length * PAGE_SIZE; - const out = new Uint8Array(totalLen); - let off = 0; - out.set(page0, off); off += PAGE_SIZE; - for (const c of chains) { - for (const p of c.pages) { - if (p.length !== PAGE_SIZE) throw new Error("page size mismatch"); - out.set(p, off); off += PAGE_SIZE; - } - } - - await mkdir(dirname(path), { recursive: true }); - await writeFile(path, out); -} diff --git a/maple-ts/src/scan.ts b/maple-ts/src/scan.ts deleted file mode 100644 index d8bfba3..0000000 --- a/maple-ts/src/scan.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { readdir } from "node:fs/promises"; -import { join, extname, basename, dirname, isAbsolute, resolve } from "node:path"; - -const AUDIO_EXTS = new Set([".mp3", ".flac", ".wav", ".aiff", ".aif", ".m4a"]); - -async function walk(dir: string): Promise { - const out: string[] = []; - const entries = await readdir(dir, { withFileTypes: true }); - for (const e of entries) { - const p = join(dir, e.name); - if (e.isDirectory()) { - out.push(...(await walk(p))); - } else if (e.isFile()) { - out.push(p); - } - } - return out; -} - -export async function scanAudio(inputDir: string): Promise { - const all = await walk(inputDir); - return all.filter((p) => AUDIO_EXTS.has(extname(p).toLowerCase())).sort(); -} - -export async function parseM3u8(playlistPath: string): Promise { - const base = dirname(playlistPath); - const text = await Bun.file(playlistPath).text(); - const entries: string[] = []; - for (const raw of text.split(/\r?\n/)) { - const s = raw.trim(); - if (!s || s.startsWith("#")) continue; - entries.push(isAbsolute(s) ? resolve(s) : resolve(join(base, s))); - } - return entries; -} - -export async function scanPlaylists(inputDir: string): Promise> { - const all = await walk(inputDir); - const playlists = all.filter((p) => extname(p).toLowerCase() === ".m3u8").sort(); - const out = new Map(); - for (const p of playlists) { - const stem = basename(p, extname(p)); - out.set(stem, await parseM3u8(p)); - } - return out; -} diff --git a/maple-ts/tests/strings.test.ts b/maple-ts/tests/strings.test.ts deleted file mode 100644 index d326ffb..0000000 --- a/maple-ts/tests/strings.test.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { expect, test } from "bun:test"; -import { encodeDevicesql } from "../src/pdb/strings.ts"; - -test("empty encodes to 0x03", () => { - expect(Array.from(encodeDevicesql(""))).toEqual([0x03]); -}); - -test("short ascii uses short form", () => { - const out = encodeDevicesql("ABC"); - // length = 3+1 = 4; (4<<1)|1 = 9 - expect(out[0]).toBe(9); - expect(Array.from(out.slice(1))).toEqual([0x41, 0x42, 0x43]); -}); - -test("non-ascii uses utf-16 form", () => { - const out = encodeDevicesql("é"); - expect(out[0]).toBe(0x90); - // total_length = utf16 bytes (2) + 4 = 6 - expect(out[1]).toBe(6); - expect(out[2]).toBe(0); - expect(out[3]).toBe(0); - expect(out[4]).toBe(0xe9); - expect(out[5]).toBe(0x00); -}); diff --git a/maple-ts/tests/writer_smoke.test.ts b/maple-ts/tests/writer_smoke.test.ts deleted file mode 100644 index fde99df..0000000 --- a/maple-ts/tests/writer_smoke.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { expect, test } from "bun:test"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { newDatabase, newTrack } from "../src/model.ts"; -import { writePdb } from "../src/pdb/writer.ts"; -import { PAGE_SIZE } from "../src/pdb/pager.ts"; - -test("writes a multi-page-aligned pdb file", async () => { - const dir = await mkdtemp(join(tmpdir(), "maple-ts-")); - try { - const db = newDatabase(); - db.artists.push({ id: 1, name: "A" }); - db.albums.push({ id: 1, name: "Alb", artist_id: 1 }); - db.tracks.push(newTrack({ - id: 1, title: "T", artist_id: 1, album_id: 1, - filename: "t.mp3", file_path: "/Contents/A/Alb/t.mp3", - })); - db.playlist_nodes.push({ id: 1, parent_id: 0, sort_order: 1, name: "P", is_folder: false }); - db.playlist_entries.push({ playlist_id: 1, track_id: 1, entry_index: 1 }); - - const out = join(dir, "export.pdb"); - await writePdb(db, out); - const buf = await readFile(out); - expect(buf.length % PAGE_SIZE).toBe(0); - expect(buf.length).toBeGreaterThanOrEqual(PAGE_SIZE * 6); - } finally { - await rm(dir, { recursive: true, force: true }); - } -}); diff --git a/maple-ts/tsconfig.json b/maple-ts/tsconfig.json deleted file mode 100644 index bacc0c6..0000000 --- a/maple-ts/tsconfig.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "bundler", - "types": ["bun-types"], - "strict": true, - "esModuleInterop": true, - "allowSyntheticDefaultImports": true, - "skipLibCheck": true, - "noEmit": true, - "verbatimModuleSyntax": false - }, - "include": ["src/**/*.ts", "tests/**/*.ts"] -} diff --git a/maple.sublime-project b/maple.sublime-project index cb97cdb..700eef6 100755 --- a/maple.sublime-project +++ b/maple.sublime-project @@ -1,36 +1,54 @@ -{ - "folders": - [ - { - "path": ".", - }, - { - "path": "h:ome\\ntr\\code\\rekordcrate", - }, - { - "path": "U:sers\\nathanrashleigh\\code\\code\\rekordcrate", - }, - { - "path": "C:\\Users\\ntr\\code\\rekordcrate" - }, - ], - "settings": - { - "cargo_build": - { - "paths": - { - "C:\\Users\\ntr\\code\\maple\\maple-rs": - { - "variants": - { - "run": - { - "target": "--bin maple", - }, - }, - }, - }, - }, - }, -} +{ + "folders": + [ + { + "path": ".", + }, + ], + "settings": + { + "LSP": + { + "LSP-pyright": + { + // Use maple-py/ as the python project root so pyrightconfig.json + // there is picked up (extraPaths=["src"], venvPath=".venv", + // pythonVersion="3.14"). The interpreter is the uv-managed venv. + "settings": + { + "python.pythonPath": "$folder/maple-py/.venv/bin/python", + "python.analysis.extraPaths": + [ + "$folder/maple-py/src", + ], + "python.analysis.typeCheckingMode": "basic", + "python.analysis.diagnosticMode": "openFilesOnly", + "python.analysis.autoImportCompletions": true, + "python.analysis.indexing": true, + }, + // Pin the LSP server's working directory to maple-py/ so file + // lookups resolve relative to where pyrightconfig.json lives. + "initializationOptions": + { + "pythonPath": "$folder/maple-py/.venv/bin/python", + }, + }, + }, + "cargo_build": + { + "paths": + { + "$folder/maple-rs": + { + "variants": + { + "run": + { + "target": "--bin maple", + }, + }, + }, + }, + }, + }, +}