claude idiot
This commit is contained in:
Nathan Rashleigh committed
1 parent
8b4ac2cdaf
commit
ea1669babf
32 files changed
+1808
-1150
No files matched your search
@@ -14,6 +14,7 @@ update m3u8 matching files in spotify to files in collection
|
||||
|
||||
### prepare for decks
|
||||
|
||||
store ANLZ in metadata directory by isrc
|
||||
create data/usb directory
|
||||
convert to mp3 v0 jstereo
|
||||
- output into the data/usb directory
|
||||
|
||||
@@ -40,13 +40,50 @@ sockseek input:
|
||||
collection ids="7f5tPIfj21UJI4zfDmH4rx 6sAl8LhU6ediFNJRuQQIQX":
|
||||
cd maple-py && uv run python -m maple.collection {{ids}}
|
||||
|
||||
# Build a rekordbox USB layout at <dst> from the collection at <src>.
|
||||
# Converts FLAC→MP3 V0, extracts artwork, writes ANLZ (with PWAV/PWV3
|
||||
# blue/white waveforms) and PIONEER/rekordbox/export.pdb.
|
||||
export src="data/collection" dst="data/usb":
|
||||
cd maple-py && uv run python -m maple "{{justfile_directory() / src}}" "{{justfile_directory() / dst}}"
|
||||
|
||||
sync args:
|
||||
s5cmd {{args}}
|
||||
|
||||
usb dst="/Volumes/MAPLE":
|
||||
COPYFILE_DISABLE=1 rsync -rtvh --delete --size-only --no-perms --no-owner --no-group --modify-window=2 maple-py/data/usb/Contents/ {{dst}}/Contents/
|
||||
COPYFILE_DISABLE=1 rsync -rtvh --delete --size-only --no-perms --no-owner --no-group --modify-window=2 --exclude='USBANLZ/' maple-py/data/usb/PIONEER/ {{dst}}/PIONEER/
|
||||
rm -f {{dst}}/PIONEER/rekordbox/RBFLTR.DAT
|
||||
# Sync data/usb/ → mounted USB stick (default /media/ntr/MAPLE), then
|
||||
# safely unmount so the stick can be pulled without risk to the FAT
|
||||
# write-back cache.
|
||||
usb dst="/media/ntr/MAPLE":
|
||||
@test -d {{dst}} || (echo "USB not mounted at {{dst}}"; exit 1)
|
||||
rsync -rtvh --delete --size-only --no-perms --no-owner --no-group --modify-window=2 "{{justfile_directory()}}/data/usb/" "{{dst}}/"
|
||||
sync
|
||||
@# Try udisksctl first (no sudo, handles the udev event); fall back to
|
||||
@# plain `umount` if udisks isn't available. Either way the recipe
|
||||
@# exits 0 even if the unmount fails (e.g. someone has a file open).
|
||||
@udisksctl unmount -b "$(findmnt -no SOURCE {{dst}})" 2>/dev/null && echo "unmounted {{dst}}" \
|
||||
|| (umount {{dst}} 2>/dev/null && echo "unmounted {{dst}}") \
|
||||
|| echo "warning: could not unmount {{dst}} — close any file browsers + retry"
|
||||
|
||||
# Build a FAT32 image of data/usb/ — ready to dd onto a stick.
|
||||
# `size` must exceed the data/usb/ tree (rough rule: data size + ~50 MB FAT overhead).
|
||||
usb-image size="2G" img="data/maple.img":
|
||||
@command -v mcopy >/dev/null || (echo "mtools not installed (apt install mtools)"; exit 1)
|
||||
@test -d "{{justfile_directory()}}/data/usb" || (echo "no data/usb (run \`just export\` first)"; exit 1)
|
||||
rm -f "{{justfile_directory()}}/{{img}}"
|
||||
truncate -s {{size}} "{{justfile_directory()}}/{{img}}"
|
||||
mkfs.vfat -F 32 -n MAPLE "{{justfile_directory()}}/{{img}}"
|
||||
cd "{{justfile_directory()}}/data/usb" && mcopy -i "{{justfile_directory()}}/{{img}}" -s -Q -p -m * ::/
|
||||
@echo "image ready:" && ls -lh "{{justfile_directory()}}/{{img}}"
|
||||
|
||||
# DESTRUCTIVE: dd data/maple.img onto a block device (e.g. /dev/sdb).
|
||||
# Double-check `lsblk` first — wrong device wipes that drive.
|
||||
[confirm("usb-flash will OVERWRITE the entire device. Proceed?")]
|
||||
usb-flash dev img="data/maple.img":
|
||||
@test -b {{dev}} || (echo "{{dev}} is not a block device"; exit 1)
|
||||
@test -f "{{justfile_directory()}}/{{img}}" || (echo "no image at {{img}}; run \`just usb-image\` first"; exit 1)
|
||||
@lsblk -no NAME,SIZE,LABEL,MODEL {{dev}}
|
||||
sudo dd if="{{justfile_directory()}}/{{img}}" of={{dev}} bs=4M status=progress oflag=direct conv=fsync
|
||||
sudo sync
|
||||
@echo "done — safe to remove (try: udisksctl power-off -b {{dev}})"
|
||||
|
||||
usb-pdb dst="/Volumes/MAPLE":
|
||||
@test -d {{dst}}/PIONEER/rekordbox || (echo "no rekordbox dir at {{dst}}/PIONEER"; exit 1)
|
||||
|
||||
@@ -9,10 +9,12 @@ authors = [
|
||||
requires-python = ">=3.14"
|
||||
dependencies = [
|
||||
"aiohttp>=3.10",
|
||||
"aubio>=0.4.9",
|
||||
"kaitaistruct>=0.11",
|
||||
"loguru>=0.7",
|
||||
"mutagen>=1.47.0",
|
||||
"pillow>=12.2.0",
|
||||
"polars>=1.0",
|
||||
"pyrekordbox>=0.4.4",
|
||||
]
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"include": [
|
||||
"src",
|
||||
"tests"
|
||||
],
|
||||
"extraPaths": [
|
||||
"src"
|
||||
],
|
||||
"exclude": [
|
||||
"**/__pycache__",
|
||||
"**/.venv",
|
||||
"data",
|
||||
"re"
|
||||
],
|
||||
"venvPath": ".",
|
||||
"venv": ".venv",
|
||||
"pythonVersion": "3.14",
|
||||
"pythonPlatform": "Linux",
|
||||
"typeCheckingMode": "basic",
|
||||
"reportMissingImports": "error",
|
||||
"reportMissingModuleSource": "warning",
|
||||
"useLibraryCodeForTypes": true
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
"""Audio analysis: analytical BPM detection + waveform envelope extraction.
|
||||
|
||||
ffmpeg decodes the source to 44.1 kHz mono float32 PCM on stdout. The
|
||||
samples are then either streamed in hop-size chunks into aubio.tempo for
|
||||
BPM detection, or fully buffered and binned to produce the per-column
|
||||
amplitude envelopes that PWAV / PWV3 ANLZ sections embed.
|
||||
|
||||
The decoded audio never touches disk — ffmpeg's `-f f32le` output is
|
||||
piped straight to numpy.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from base64 import b64decode, b64encode
|
||||
from pathlib import Path
|
||||
|
||||
import aubio
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
from loguru import logger
|
||||
|
||||
HOP_SIZE = 512
|
||||
WIN_SIZE = 1024
|
||||
SAMPLE_RATE = 44100
|
||||
MIN_BEATS = 8
|
||||
|
||||
# Waveform envelope dimensions.
|
||||
PREVIEW_COLS = 400 # PWAV is fixed at 400 columns (1 byte/col).
|
||||
SCROLL_FPS = 150 # PWV3 = ~150 columns/sec across the whole track.
|
||||
PWAV_LEVELS = 31 # PWAV/PWV3 stores 5-bit heights (bit 5 = white flag).
|
||||
|
||||
|
||||
def _stream_pcm(path: Path):
|
||||
"""Yield successive hop-sized float32 numpy arrays of mono PCM.
|
||||
|
||||
Pads the final chunk with zeros so every yield is exactly HOP_SIZE samples
|
||||
(aubio.tempo requires fixed-size inputs).
|
||||
"""
|
||||
proc = subprocess.Popen(
|
||||
["ffmpeg", "-nostdin", "-v", "error",
|
||||
"-i", str(path),
|
||||
"-f", "f32le", "-ac", "1", "-ar", str(SAMPLE_RATE), "-"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
assert proc.stdout is not None
|
||||
bytes_per_chunk = HOP_SIZE * 4 # float32
|
||||
while True:
|
||||
buf = proc.stdout.read(bytes_per_chunk)
|
||||
if not buf:
|
||||
break
|
||||
arr = np.frombuffer(buf, dtype=np.float32)
|
||||
if len(arr) < HOP_SIZE:
|
||||
pad = np.zeros(HOP_SIZE - len(arr), dtype=np.float32)
|
||||
arr = np.concatenate([arr, pad])
|
||||
yield arr
|
||||
finally:
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
proc.wait()
|
||||
|
||||
|
||||
def detect_bpm(path: Path) -> float | None:
|
||||
"""Return the analytical BPM of `path`, or None on failure.
|
||||
|
||||
Uses median-of-intervals (via polars), which is robust against
|
||||
aubio's "live" running-average swings on transient sections.
|
||||
"""
|
||||
if not shutil.which("ffmpeg"):
|
||||
logger.warning("ffmpeg not on PATH; BPM detection disabled")
|
||||
return None
|
||||
|
||||
tempo = aubio.tempo("default", WIN_SIZE, HOP_SIZE, SAMPLE_RATE)
|
||||
beats: list[float] = []
|
||||
try:
|
||||
for chunk in _stream_pcm(path):
|
||||
if tempo(chunk):
|
||||
beats.append(tempo.get_last_s())
|
||||
except Exception as e:
|
||||
logger.warning("BPM detection failed for {}: {}", path.name, e)
|
||||
return None
|
||||
|
||||
if len(beats) < MIN_BEATS:
|
||||
return None
|
||||
intervals = pl.Series(
|
||||
[beats[i + 1] - beats[i] for i in range(len(beats) - 1)]
|
||||
)
|
||||
median_interval = intervals.median()
|
||||
if not median_interval or median_interval <= 0:
|
||||
return None
|
||||
return 60.0 / median_interval
|
||||
|
||||
|
||||
def _decode_pcm(path: Path) -> np.ndarray:
|
||||
"""Decode `path` to a 1D float32 numpy array of mono PCM at SAMPLE_RATE.
|
||||
|
||||
Returns an empty array on ffmpeg failure.
|
||||
"""
|
||||
if not shutil.which("ffmpeg"):
|
||||
logger.warning("ffmpeg not on PATH; cannot extract waveform")
|
||||
return np.empty(0, dtype=np.float32)
|
||||
proc = subprocess.run(
|
||||
["ffmpeg", "-nostdin", "-v", "error",
|
||||
"-i", str(path),
|
||||
"-f", "f32le", "-ac", "1", "-ar", str(SAMPLE_RATE), "-"],
|
||||
capture_output=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
logger.warning("ffmpeg failed for {}: {}", path.name,
|
||||
proc.stderr.decode("utf-8", "replace").strip())
|
||||
return np.empty(0, dtype=np.float32)
|
||||
return np.frombuffer(proc.stdout, dtype=np.float32)
|
||||
|
||||
|
||||
def _bin_amplitudes(samples: np.ndarray, n_bins: int, levels: int = PWAV_LEVELS) -> bytes:
|
||||
"""Split `samples` into `n_bins` contiguous windows, take the peak |s| per
|
||||
window, quantise to 0..levels, and return one byte per bin.
|
||||
|
||||
Quantising at 5-bit resolution matches PWAV's `bits 0-4 = height` encoding;
|
||||
bit 5 (the "white overshoot" flag) is left at 0 for now — most CDJ firmware
|
||||
is happy without it.
|
||||
"""
|
||||
if n_bins <= 0 or len(samples) == 0:
|
||||
return bytes(n_bins)
|
||||
chunks = np.array_split(np.abs(samples), n_bins)
|
||||
peaks = np.array([float(c.max()) if len(c) else 0.0 for c in chunks])
|
||||
peak_max = float(peaks.max()) if peaks.size else 0.0
|
||||
if peak_max <= 0:
|
||||
return bytes(n_bins)
|
||||
q = np.clip(np.round(peaks / peak_max * levels), 0, levels).astype(np.uint8)
|
||||
return q.tobytes()
|
||||
|
||||
|
||||
def extract_waveform_envelope(path: Path) -> dict[str, bytes]:
|
||||
"""Return the two byte streams a PWAV + PWV3 pair needs.
|
||||
|
||||
Output:
|
||||
{
|
||||
"preview": bytes of length PREVIEW_COLS (always 400),
|
||||
"scroll": bytes of length ~ duration * SCROLL_FPS,
|
||||
}
|
||||
On decode failure both streams are empty.
|
||||
"""
|
||||
samples = _decode_pcm(path)
|
||||
if samples.size == 0:
|
||||
return {"preview": b"", "scroll": b""}
|
||||
duration = samples.size / SAMPLE_RATE
|
||||
scroll_cols = max(1, int(round(duration * SCROLL_FPS)))
|
||||
return {
|
||||
"preview": _bin_amplitudes(samples, PREVIEW_COLS),
|
||||
"scroll": _bin_amplitudes(samples, scroll_cols),
|
||||
}
|
||||
|
||||
|
||||
def _detect_beats_in_samples(samples: np.ndarray) -> list[float]:
|
||||
"""Run aubio's onset detector across a pre-loaded mono PCM buffer.
|
||||
|
||||
Returns the timestamps (seconds) of every detected beat.
|
||||
"""
|
||||
tempo = aubio.tempo("default", WIN_SIZE, HOP_SIZE, SAMPLE_RATE)
|
||||
beats: list[float] = []
|
||||
for i in range(0, len(samples), HOP_SIZE):
|
||||
chunk = samples[i:i + HOP_SIZE]
|
||||
if len(chunk) < HOP_SIZE:
|
||||
chunk = np.concatenate(
|
||||
[chunk, np.zeros(HOP_SIZE - len(chunk), dtype=np.float32)]
|
||||
)
|
||||
if tempo(chunk):
|
||||
beats.append(tempo.get_last_s())
|
||||
return beats
|
||||
|
||||
|
||||
def _project_beat_grid(beats: list[float], bpm: float, duration_s: float) -> list[tuple[int, int, int]]:
|
||||
"""Project a rigid `bpm` beat grid using the first detected beat as anchor.
|
||||
|
||||
Returns `[(beat_number 1-4, tempo*100, time_ms), ...]` covering t=0 to
|
||||
`duration_s`. Rigid grids are what CDJs prefer over noisy per-beat
|
||||
timing — the strongest detected beat fixes the phase and we project
|
||||
forward + backward at the canonical tempo.
|
||||
"""
|
||||
if not beats or bpm <= 0:
|
||||
return []
|
||||
interval = 60.0 / bpm
|
||||
first = beats[0]
|
||||
# Walk back from the first detected beat to t≥0 so the grid covers the intro.
|
||||
n_before = int(first / interval)
|
||||
anchor = first - n_before * interval
|
||||
if anchor < 0:
|
||||
anchor += interval
|
||||
n_before -= 1
|
||||
tempo_x100 = round(bpm * 100)
|
||||
grid: list[tuple[int, int, int]] = []
|
||||
t = anchor
|
||||
i = 0
|
||||
while t <= duration_s + 1.0:
|
||||
time_ms = int(round(t * 1000))
|
||||
beat_number = (i % 4) + 1
|
||||
grid.append((beat_number, tempo_x100, time_ms))
|
||||
i += 1
|
||||
t = anchor + i * interval
|
||||
return grid
|
||||
|
||||
|
||||
CACHE_VERSION = 1
|
||||
|
||||
|
||||
def _sanitise_isrc(isrc: str) -> str:
|
||||
"""Reduce an ISRC to a filename-safe form (uppercase, alnum only)."""
|
||||
return "".join(c for c in (isrc or "").upper() if c.isalnum())
|
||||
|
||||
|
||||
def cache_path_for(cache_dir: Path, isrc: str) -> Path | None:
|
||||
"""Return the path the cache file for `isrc` lives at, or None if no isrc."""
|
||||
s = _sanitise_isrc(isrc)
|
||||
if not s:
|
||||
return None
|
||||
return cache_dir / f"{s}.json"
|
||||
|
||||
|
||||
def save_analysis(cache_dir: Path, isrc: str, analysis: dict) -> Path | None:
|
||||
"""Persist `analysis` for `isrc` under `cache_dir`. No-op if isrc empty."""
|
||||
target = cache_path_for(cache_dir, isrc)
|
||||
if target is None:
|
||||
return None
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"version": CACHE_VERSION,
|
||||
"isrc": _sanitise_isrc(isrc),
|
||||
"bpm": analysis.get("bpm"),
|
||||
"beats": analysis.get("beats", []),
|
||||
"preview_b64": b64encode(analysis.get("preview") or b"").decode("ascii"),
|
||||
"scroll_b64": b64encode(analysis.get("scroll") or b"").decode("ascii"),
|
||||
}
|
||||
target.write_text(json.dumps(payload, separators=(",", ":")))
|
||||
return target
|
||||
|
||||
|
||||
def load_analysis(cache_dir: Path, isrc: str) -> dict | None:
|
||||
"""Return cached analysis for `isrc`, or None if absent/stale/unreadable."""
|
||||
target = cache_path_for(cache_dir, isrc)
|
||||
if target is None or not target.exists():
|
||||
return None
|
||||
try:
|
||||
d = json.loads(target.read_text())
|
||||
if d.get("version") != CACHE_VERSION:
|
||||
return None
|
||||
return {
|
||||
"bpm": d.get("bpm"),
|
||||
# JSON tuples come back as lists; PQTZ writer accepts either.
|
||||
"beats": [tuple(b) for b in d.get("beats", [])],
|
||||
"preview": b64decode(d.get("preview_b64") or ""),
|
||||
"scroll": b64decode(d.get("scroll_b64") or ""),
|
||||
}
|
||||
except (json.JSONDecodeError, OSError, ValueError) as e:
|
||||
logger.warning("analysis cache read failed for {}: {}", isrc, e)
|
||||
return None
|
||||
|
||||
|
||||
def analyze_audio(path: Path) -> dict:
|
||||
"""One-pass decode + analysis. Returns:
|
||||
{
|
||||
"bpm": float | None, # median-of-intervals tempo
|
||||
"beats": list[(beat_n, tempo_x100, time_ms)], # for PQTZ
|
||||
"preview": bytes, # for PWAV (400-byte amplitude strip)
|
||||
"scroll": bytes, # for PWV3 (~150 cols/sec)
|
||||
}
|
||||
"""
|
||||
samples = _decode_pcm(path)
|
||||
out: dict = {"bpm": None, "beats": [], "preview": b"", "scroll": b""}
|
||||
if samples.size == 0:
|
||||
return out
|
||||
|
||||
# Beat onsets.
|
||||
beat_times = _detect_beats_in_samples(samples)
|
||||
if len(beat_times) >= MIN_BEATS:
|
||||
intervals = pl.Series(
|
||||
[beat_times[i + 1] - beat_times[i] for i in range(len(beat_times) - 1)]
|
||||
)
|
||||
median_interval = intervals.median()
|
||||
if median_interval and median_interval > 0:
|
||||
out["bpm"] = 60.0 / median_interval
|
||||
duration_s = samples.size / SAMPLE_RATE
|
||||
out["beats"] = _project_beat_grid(beat_times, out["bpm"], duration_s)
|
||||
|
||||
# Waveform envelopes.
|
||||
duration = samples.size / SAMPLE_RATE
|
||||
scroll_cols = max(1, int(round(duration * SCROLL_FPS)))
|
||||
out["preview"] = _bin_amplitudes(samples, PREVIEW_COLS)
|
||||
out["scroll"] = _bin_amplitudes(samples, scroll_cols)
|
||||
return out
|
||||
+240
-21
@@ -1,41 +1,260 @@
|
||||
"""Generate a minimal rekordbox ANLZ0000.DAT analysis file.
|
||||
"""Generate a rekordbox ANLZ0000.{DAT,EXT} pair.
|
||||
|
||||
The CDJ/XDJ reads the path from a `PPTH` section to confirm which audio
|
||||
file the analysis applies to. We emit just that section -- no beat grid,
|
||||
no waveforms -- which is enough for the player to consider the track's
|
||||
analysis "present" and surface the track in the library. Without an
|
||||
analyze file at the path the pdb references, some firmware hides the
|
||||
track entirely.
|
||||
ANLZ0000.DAT — what CDJ-900-era through CDJ-3000 firmware reads to display
|
||||
the loaded-deck BPM, the touch-strip preview waveform, and the cue list:
|
||||
|
||||
PPTH player-relative audio path
|
||||
PVBR variable-bitrate seek index (zero-filled placeholder; matches the
|
||||
shape rekordbox writes for files where the seek index isn't useful)
|
||||
PQTZ beat grid (per-beat: beat_number, tempo*100, time_ms)
|
||||
PWAV 400-byte blue/white touch-strip preview
|
||||
PWV2 100-byte "tiny" overview waveform
|
||||
PCOB memory-cue list (empty placeholder)
|
||||
PCOB hot-cue list (empty placeholder)
|
||||
|
||||
ANLZ0000.EXT — extended data; we write a minimal PPTH-only file so newer
|
||||
firmware that opens .EXT as part of its load flow finds a valid container.
|
||||
Color / 3-band waveforms (PWV4/PWV5/PWV6/PWV7) belong in .EXT/.2EX and
|
||||
aren't generated yet.
|
||||
|
||||
Field discoveries documented by byte-diffing maple's output against a real
|
||||
rekordbox export at data/q1-rekordbox/.
|
||||
|
||||
All multi-byte fields in ANLZ files are BIG-endian, unlike the pdb.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from struct import pack
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from maple.analyze import analyze_audio, load_analysis, save_analysis
|
||||
|
||||
# Real rekordbox PMAI header carries three non-zero u4 markers in the bytes
|
||||
# after the standard PMAI/len_header/len_file triple. Verified by hex-dump
|
||||
# of data/q1-rekordbox/PIONEER/USBANLZ/P020/0002E10A/ANLZ0000.{DAT,EXT,2EX}
|
||||
# — all three files share the same 16-byte trailer.
|
||||
PMAI_HEADER_TRAILER = pack(">IIII", 0x00000001, 0x00010000, 0x00010000, 0x00000000)
|
||||
ANLZ_HEADER_LEN = 28 # PMAI + 2 size u4s + the 16-byte trailer above
|
||||
|
||||
|
||||
def _section_pth(path: str) -> bytes:
|
||||
"""PPTH section: u4 len_path, then UTF-16BE bytes (null-terminated)."""
|
||||
utf16 = path.encode("utf-16-be") + b"\x00\x00"
|
||||
len_path = len(utf16)
|
||||
body = pack(">I", len_path) + utf16
|
||||
# Section header: 4 fourcc + u4 len_header + u4 len_tag
|
||||
fourcc = b"PPTH"
|
||||
len_header = 16 # section header size (incl. fourcc + 2 u4s + len_path u4)
|
||||
len_tag = 12 + len(body) # full section size
|
||||
len_header = 16 # fourcc + 2 u4 size fields + len_path u4
|
||||
len_tag = 12 + len(body)
|
||||
return fourcc + pack(">II", len_header, len_tag) + body
|
||||
|
||||
|
||||
def build_minimal_anlz(audio_path: str) -> bytes:
|
||||
"""Return bytes of a minimal ANLZ0000.DAT pointing at `audio_path`."""
|
||||
pth = _section_pth(audio_path)
|
||||
header_len = 28
|
||||
file_len = header_len + len(pth)
|
||||
header = b"PMAI" + pack(">II", header_len, file_len)
|
||||
header += b"\x00" * (header_len - len(header))
|
||||
assert len(header) == header_len
|
||||
return header + pth
|
||||
def _section_pvbr() -> bytes:
|
||||
"""PVBR section: 400-entry VBR seek index. We write a zero-filled
|
||||
placeholder — rekordbox accepts this for fixed-bitrate sources and it
|
||||
matches the section's presence in real exports."""
|
||||
fourcc = b"PVBR"
|
||||
len_header = 16 # prelude(12) + u4 unnamed0(4)
|
||||
# 400 u4 seek index + 4 trailer bytes = 1604 bytes of variable data
|
||||
# (verified by len_tag=1620 in the real Q1.1 export).
|
||||
body_var = b"\x00" * 1604
|
||||
unnamed0 = b"\x00\x00\x00\x00"
|
||||
len_tag = len_header + len(body_var) # 16 + 1604 = 1620
|
||||
return fourcc + pack(">II", len_header, len_tag) + unnamed0 + body_var
|
||||
|
||||
|
||||
def write_anlz(audio_path: str, dest: Path) -> None:
|
||||
def _section_pqtz(beats: list[tuple[int, int, int]]) -> bytes:
|
||||
"""PQTZ: BeatGridTag — per-beat (beat_number, tempo×100, time_ms).
|
||||
|
||||
The CDJ's loaded-deck BPM readout, tempo slider and sync engine all
|
||||
read from this section. Real rekordbox sets `_unnamed1=0x00080000`
|
||||
(was 0 in maple's earlier writer — likely the cause of the deck-side
|
||||
BPM display staying blank even when the grid was otherwise valid).
|
||||
"""
|
||||
fourcc = b"PQTZ"
|
||||
len_header = 24 # prelude(12) + 3 u4 body-header fields
|
||||
num_beats = len(beats)
|
||||
body = pack(">III", 0x00000000, 0x00080000, num_beats)
|
||||
for bn, tempo_x100, t_ms in beats:
|
||||
body += pack(">HHI", bn, tempo_x100, t_ms)
|
||||
len_tag = 12 + len(body)
|
||||
return fourcc + pack(">II", len_header, len_tag) + body
|
||||
|
||||
|
||||
def _section_pwav(preview: bytes) -> bytes:
|
||||
"""PWAV: WavePreviewTag — 400-byte blue/white touch-strip preview.
|
||||
Each byte = (white_flag << 5) | height (bits 0-4)."""
|
||||
fourcc = b"PWAV"
|
||||
len_header = 20 # prelude(12) + len_data(4) + _unnamed1(4)
|
||||
len_tag = len_header + len(preview)
|
||||
return fourcc + pack(">IIII", len_header, len_tag, len(preview), 0x00010000) + preview
|
||||
|
||||
|
||||
def _section_pwv2(tiny: bytes) -> bytes:
|
||||
"""PWV2: 100-byte "tiny" overview waveform. Same byte encoding as PWAV,
|
||||
just at a coarser resolution. Real Q1.1 export has this section."""
|
||||
fourcc = b"PWV2"
|
||||
len_header = 20 # prelude(12) + len_data(4) + _unnamed1(4)
|
||||
len_tag = len_header + len(tiny)
|
||||
return fourcc + pack(">IIII", len_header, len_tag, len(tiny), 0x00010000) + tiny
|
||||
|
||||
|
||||
def _section_pcob(cue_list_type: int) -> bytes:
|
||||
"""PCOB: CueTag — cue/loop list. Real exports always include both
|
||||
memory-cue (type=0) and hot-cue (type=1) sections, even when both are
|
||||
empty. Body: u4 type, 2 bytes unnamed, u2 num_cues, u4 memory_count."""
|
||||
fourcc = b"PCOB"
|
||||
len_header = 24 # prelude(12) + body fixed prefix(12)
|
||||
body = pack(">I", cue_list_type) + b"\x00\x00" + pack(">HI", 0, 0)
|
||||
len_tag = 12 + len(body)
|
||||
return fourcc + pack(">II", len_header, len_tag) + body
|
||||
|
||||
|
||||
def _section_pwv3(scroll: bytes) -> bytes:
|
||||
"""PWV3: WaveScrollTag — full-track mono scroll waveform.
|
||||
|
||||
NOT written into ANLZ0000.DAT (real rekordbox places PWV3 in .EXT, not
|
||||
.DAT). Kept here for reference / future .EXT generation."""
|
||||
fourcc = b"PWV3"
|
||||
len_header = 24
|
||||
len_entry_bytes = 1
|
||||
len_entries = len(scroll)
|
||||
len_tag = len_header + len(scroll)
|
||||
return fourcc + pack(">IIIII", len_header, len_tag, len_entry_bytes, len_entries, 0x00960000) + scroll
|
||||
|
||||
|
||||
def _bin_tiny_from_preview(preview: bytes, cols: int = 100) -> bytes:
|
||||
"""Derive the PWV2 tiny envelope from PWAV's 400-byte preview by
|
||||
re-binning groups of 4 columns to their per-bin max."""
|
||||
if not preview:
|
||||
return b""
|
||||
chunk = max(1, len(preview) // cols)
|
||||
out = bytearray(cols)
|
||||
for i in range(cols):
|
||||
slc = preview[i * chunk : (i + 1) * chunk] if i < cols - 1 else preview[i * chunk:]
|
||||
if slc:
|
||||
# Per-byte max ignoring the white-flag bit, then put it back at
|
||||
# the corresponding compressed amplitude.
|
||||
heights = [b & 0x1F for b in slc]
|
||||
out[i] = max(heights) & 0x1F
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def _pmai_header(file_len: int) -> bytes:
|
||||
"""28-byte PMAI header — fourcc + sizes + the canonical trailer that
|
||||
rekordbox writes (was zero-padded in maple's earlier output)."""
|
||||
return b"PMAI" + pack(">II", ANLZ_HEADER_LEN, file_len) + PMAI_HEADER_TRAILER
|
||||
|
||||
|
||||
def build_dat(rb_path: str, analysis: dict | None) -> bytes:
|
||||
"""Build ANLZ0000.DAT — the file every CDJ reads on track load.
|
||||
|
||||
Section order matches rekordbox's own output: PPTH, PVBR, PQTZ, PWAV,
|
||||
PWV2, PCOB(memory), PCOB(hot).
|
||||
"""
|
||||
sections = [_section_pth(rb_path), _section_pvbr()]
|
||||
if analysis:
|
||||
if analysis.get("beats"):
|
||||
sections.append(_section_pqtz(analysis["beats"]))
|
||||
preview = analysis.get("preview") or b""
|
||||
if preview:
|
||||
sections.append(_section_pwav(preview))
|
||||
sections.append(_section_pwv2(_bin_tiny_from_preview(preview)))
|
||||
sections.append(_section_pcob(0)) # memory cues (empty)
|
||||
sections.append(_section_pcob(1)) # hot cues (empty)
|
||||
body = b"".join(sections)
|
||||
return _pmai_header(ANLZ_HEADER_LEN + len(body)) + body
|
||||
|
||||
|
||||
def build_ext(rb_path: str) -> bytes:
|
||||
"""Build a minimal ANLZ0000.EXT — PPTH-only.
|
||||
|
||||
Real rekordbox .EXT carries color waveforms and song-structure
|
||||
sections; we don't generate those yet. A bare .EXT file is enough for
|
||||
firmware that opens it as part of the standard track-load flow."""
|
||||
body = _section_pth(rb_path)
|
||||
return _pmai_header(ANLZ_HEADER_LEN + len(body)) + body
|
||||
|
||||
|
||||
def build_anlz(rb_path: str, source_audio: Path | None = None) -> bytes:
|
||||
"""Compatibility wrapper for callers that want the .DAT bytes only."""
|
||||
analysis: dict | None = None
|
||||
if source_audio is not None:
|
||||
try:
|
||||
analysis = analyze_audio(source_audio)
|
||||
except Exception as e:
|
||||
logger.warning("audio analysis failed for {}: {}", source_audio.name, e)
|
||||
return build_dat(rb_path, analysis)
|
||||
|
||||
|
||||
def build_minimal_anlz(rb_path: str) -> bytes:
|
||||
"""PPTH-only-plus-empties-DAT. Used when no source audio is available."""
|
||||
return build_dat(rb_path, analysis=None)
|
||||
|
||||
|
||||
def _isrc_for(path: Path) -> str | None:
|
||||
"""Best-effort ISRC read from a file tag (mp3/flac/m4a)."""
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(path, easy=True)
|
||||
if audio is None or audio.tags is None:
|
||||
return None
|
||||
v = audio.tags.get("isrc")
|
||||
if not v:
|
||||
return None
|
||||
return str(v[0]).strip() or None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def write_anlz(
|
||||
rb_path: str,
|
||||
dest: Path,
|
||||
source_audio: Path | None = None,
|
||||
analysis_cache_dir: Path | None = None,
|
||||
) -> dict | None:
|
||||
"""Write ANLZ0000.DAT (and a minimal ANLZ0000.EXT alongside).
|
||||
|
||||
`dest` is the .DAT path. The .EXT companion is written next to it
|
||||
automatically — every real CDJ export ships both files.
|
||||
|
||||
Returns the analysis dict (cache hit or fresh aubio run) or None when
|
||||
no source audio was given.
|
||||
"""
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(build_minimal_anlz(audio_path))
|
||||
analysis: dict | None = None
|
||||
cache_hit = False
|
||||
if source_audio is not None:
|
||||
isrc = _isrc_for(source_audio) if analysis_cache_dir else None
|
||||
if isrc and analysis_cache_dir:
|
||||
cached = load_analysis(analysis_cache_dir, isrc)
|
||||
if cached is not None:
|
||||
analysis = cached
|
||||
cache_hit = True
|
||||
logger.info("anlz: cache hit for {} (isrc {})", source_audio.name, isrc)
|
||||
if analysis is None:
|
||||
try:
|
||||
analysis = analyze_audio(source_audio)
|
||||
except Exception as e:
|
||||
logger.warning("audio analysis failed for {}: {}", source_audio.name, e)
|
||||
analysis = None
|
||||
# Populate the cache on every successful fresh analysis so subsequent
|
||||
# exports (and collection runs) can skip the ffmpeg+aubio pass. The
|
||||
# cache is the single source of truth — populated by whichever
|
||||
# pipeline (`just collection` or `just export`) sees a track first.
|
||||
if analysis is not None and not cache_hit and analysis_cache_dir is not None and isrc:
|
||||
try:
|
||||
cp = save_analysis(analysis_cache_dir, isrc, analysis)
|
||||
if cp is not None:
|
||||
logger.info("anlz: cached analysis for {} → {}", source_audio.name, cp)
|
||||
except Exception as e:
|
||||
logger.warning("could not write analysis cache for {}: {}", source_audio.name, e)
|
||||
|
||||
dest.write_bytes(build_dat(rb_path, analysis))
|
||||
# .EXT sister file. Newer firmware (esp. CDJ-3000) opens this during the
|
||||
# track-load flow; absence of the file has been observed to silently
|
||||
# disable the loaded-deck BPM/waveform display even when .DAT is intact.
|
||||
ext_path = dest.with_suffix(".EXT")
|
||||
ext_path.write_bytes(build_ext(rb_path))
|
||||
return analysis
|
||||
+289
-20
@@ -1,34 +1,116 @@
|
||||
"""maple CLI: scan a directory of audio + m3u8 playlists and build a rekordbox USB."""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from maple.anlz import write_anlz
|
||||
from maple.artwork import _bucket as _artwork_bucket
|
||||
from maple.artwork import extract_artwork
|
||||
from maple.convert import convert_flac_to_mp3, verify_duration
|
||||
from maple.copy import stage_audio, staged_path
|
||||
from maple.log import configure as configure_logging
|
||||
from maple.meta import read_tags
|
||||
from maple.model import Album, Artist, Artwork, Database, Genre, PlaylistEntry, PlaylistNode, Track
|
||||
from maple.enrich import enrich_from_manifest, load_spotify_manifest, write_bpm_tag
|
||||
from maple.pdb.writer import write_pdb
|
||||
from maple.scan import scan_audio, scan_playlists
|
||||
from maple.validate import TrackIssue, log_issues, validate_track
|
||||
|
||||
# Genre tag delimiters seen across our sources: Spotify-derived uses
|
||||
# ';', European labels often use ',', some store '/'-separated lists.
|
||||
_GENRE_SPLIT_RE = re.compile(r"\s*[;,/]\s*")
|
||||
|
||||
|
||||
def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Database:
|
||||
def split_genre_tags(raw: str) -> list[str]:
|
||||
"""Tokenise a possibly multi-value genre string into normalised parts.
|
||||
|
||||
Each token's casing is preserved as-given. Empty parts are dropped.
|
||||
Duplicates within the same source string are collapsed (first wins).
|
||||
"""
|
||||
if not raw:
|
||||
return []
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for token in _GENRE_SPLIT_RE.split(raw):
|
||||
t = token.strip()
|
||||
if not t:
|
||||
continue
|
||||
key = t.casefold()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(t)
|
||||
return out
|
||||
|
||||
|
||||
def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> tuple[Database, set[Path]]:
|
||||
"""Build the rekordbox USB layout under `usb_root`.
|
||||
|
||||
Returns the populated `Database` plus the set of absolute paths that
|
||||
were written (audio files, ANLZ files, artwork JPEGs). The caller adds
|
||||
the pdb path and calls `prune_stale` to remove anything else under
|
||||
`usb_root` left over from a previous export.
|
||||
"""
|
||||
db = Database()
|
||||
today = date.today().isoformat()
|
||||
|
||||
artist_index: dict[str, int] = {}
|
||||
album_index: dict[tuple[int, str], int] = {}
|
||||
# Key by casefolded name so "Dub Techno" and "dub techno" collapse to one row.
|
||||
genre_index: dict[str, int] = {}
|
||||
# The user-facing genre→tracks map. Populated as tracks are processed.
|
||||
genre_to_tracks: dict[str, list[int]] = defaultdict(list)
|
||||
track_by_src: dict[Path, int] = {}
|
||||
artwork_cache: dict[str, int] = {}
|
||||
written: set[Path] = set()
|
||||
|
||||
audio_files = scan_audio(input_dir)
|
||||
logger.info("scanned {}: {} audio file(s)", input_dir, len(audio_files))
|
||||
|
||||
# ---------- Spotify enrichment (symmetric with `just collection`) -------- #
|
||||
# Per the validation convention: both pipelines must be capable of
|
||||
# bringing a track to "valid" state. Validate first; only files that
|
||||
# actually need it pay the enrichment cost (mutagen tag rewrite +
|
||||
# possible artwork fetch). Repeated exports of an already-valid
|
||||
# collection skip the enrichment loop entirely.
|
||||
by_key, by_isrc = load_spotify_manifest(input_dir / "_metadata")
|
||||
cache_dir = input_dir / "_analysis"
|
||||
if by_key or by_isrc:
|
||||
logger.info(
|
||||
"loaded spotify manifest: {} (artist,title) keys, {} ISRCs",
|
||||
len(by_key), len(by_isrc),
|
||||
)
|
||||
enriched = 0
|
||||
skipped = 0
|
||||
for p in audio_files:
|
||||
if not validate_track(p, analysis_cache_dir=cache_dir):
|
||||
skipped += 1
|
||||
continue
|
||||
if enrich_from_manifest(p, by_key, by_isrc, analysis_cache_dir=cache_dir) is not None:
|
||||
enriched += 1
|
||||
logger.info(
|
||||
"enrichment: {} enriched, {} already valid (skipped)",
|
||||
enriched, skipped,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"no spotify manifest at {} — tracks won't be Spotify-enriched. "
|
||||
"Run `just collection` first to fetch + persist the manifest.",
|
||||
input_dir / "_metadata" / "spotify-tracks.json",
|
||||
)
|
||||
|
||||
t = time.perf_counter()
|
||||
initial_meta = {p: read_tags(p) for p in audio_files}
|
||||
logger.debug("read tags in {:.2f}s", time.perf_counter() - t)
|
||||
|
||||
flac_jobs: list[tuple[Path, Path]] = []
|
||||
for original in audio_files:
|
||||
if original.suffix.lower() == ".flac":
|
||||
@@ -38,15 +120,32 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Datab
|
||||
|
||||
def _job(pair: tuple[Path, Path]) -> tuple[Path, Path, bool]:
|
||||
src, dst = pair
|
||||
return src, dst, convert_flac_to_mp3(src, dst)
|
||||
t0 = time.perf_counter()
|
||||
converted = convert_flac_to_mp3(src, dst)
|
||||
return src, dst, converted, time.perf_counter() - t0
|
||||
|
||||
if flac_jobs:
|
||||
workers = max(1, os.cpu_count() or 1)
|
||||
logger.info("converting {} flac(s) → mp3 v0 (workers={})", len(flac_jobs), workers)
|
||||
t = time.perf_counter()
|
||||
n_converted = 0
|
||||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
for src, dst, converted in ex.map(_job, flac_jobs):
|
||||
print(f"{'convert' if converted else 'cached '} {src.relative_to(input_dir)}")
|
||||
for i, (src, dst, converted, dt) in enumerate(ex.map(_job, flac_jobs), 1):
|
||||
tag = "convert" if converted else "cached "
|
||||
logger.info("[{}/{}] {} {} ({:.1f}s)", i, len(flac_jobs), tag, src.relative_to(input_dir), dt)
|
||||
if converted:
|
||||
n_converted += 1
|
||||
verify_duration(src, dst)
|
||||
logger.info(
|
||||
"flac→mp3 pass: {} converted, {} cached in {:.1f}s total",
|
||||
n_converted, len(flac_jobs) - n_converted, time.perf_counter() - t,
|
||||
)
|
||||
|
||||
for original in audio_files:
|
||||
logger.info("building track table ({} files)", len(audio_files))
|
||||
build_t0 = time.perf_counter()
|
||||
validation_issues: list[TrackIssue] = []
|
||||
for track_idx, original in enumerate(audio_files, 1):
|
||||
track_t0 = time.perf_counter()
|
||||
meta = initial_meta[original]
|
||||
artist_id = artist_index.get(meta.artist)
|
||||
if artist_id is None:
|
||||
@@ -66,19 +165,50 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Datab
|
||||
meta = read_tags(dest_abs)
|
||||
else:
|
||||
dest_abs, file_path = stage_audio(original, usb_root, meta.artist, meta.album)
|
||||
written.add(dest_abs.resolve())
|
||||
|
||||
# Re-enrich the STAGED file too — `stage_audio` and the FLAC→MP3
|
||||
# converter cache by hash, so a cached dest may still carry stale
|
||||
# tags from a previous export. Skip when the staged file already
|
||||
# passes validation so re-exports of an unchanged collection don't
|
||||
# re-mutate every track.
|
||||
if (by_key or by_isrc) and validate_track(dest_abs, analysis_cache_dir=cache_dir):
|
||||
if enrich_from_manifest(dest_abs, by_key, by_isrc, analysis_cache_dir=cache_dir) is not None:
|
||||
meta = read_tags(dest_abs)
|
||||
|
||||
next_artwork_id = len(db.artwork) + 1
|
||||
artwork_id, art_file_path = extract_artwork(dest_abs, usb_root, next_artwork_id, artwork_cache)
|
||||
if art_file_path is not None:
|
||||
db.artwork.append(Artwork(id=artwork_id, path=art_file_path))
|
||||
# Track artwork paths for the prune sweep — both this run's writes
|
||||
# and any artwork_cache hits (where extract_artwork returned None
|
||||
# for art_file_path because the jpeg already exists from a prior
|
||||
# iteration). _bucket() is the same id-to-folder mapping artwork.py
|
||||
# uses, so this rebuilds the absolute on-disk path exactly.
|
||||
if artwork_id:
|
||||
art_bucket = _artwork_bucket(artwork_id)
|
||||
for fname in (f"a{artwork_id}.jpg", f"a{artwork_id}_m.jpg"):
|
||||
written.add((usb_root / "PIONEER" / "Artwork" / art_bucket / fname).resolve())
|
||||
|
||||
genre_id = 0
|
||||
if meta.genre:
|
||||
genre_id = genre_index.get(meta.genre, 0)
|
||||
if genre_id == 0:
|
||||
genre_id = len(db.genres) + 1
|
||||
genre_index[meta.genre] = genre_id
|
||||
db.genres.append(Genre(id=genre_id, name=meta.genre))
|
||||
# Genre: split multi-value tag strings ("dub techno;minimal techno",
|
||||
# "Drum & Bass, Électronique", etc.) into individual genre rows so the
|
||||
# CDJ's Genre filter is usable. The track FK points at the FIRST
|
||||
# (most-specific) token; every other token still gets its own row in
|
||||
# the pdb's genre table so it appears in the browse list, and we
|
||||
# record the {genre→track_ids} map so we can log + reuse it later.
|
||||
primary_genre_id = 0
|
||||
tokens = split_genre_tags(meta.genre)
|
||||
for n, token in enumerate(tokens):
|
||||
key = token.casefold()
|
||||
gid = genre_index.get(key)
|
||||
if gid is None:
|
||||
gid = len(db.genres) + 1
|
||||
genre_index[key] = gid
|
||||
db.genres.append(Genre(id=gid, name=token))
|
||||
if n == 0:
|
||||
primary_genre_id = gid
|
||||
genre_to_tracks[token].append(len(db.tracks) + 1) # this track's id-to-be
|
||||
genre_id = primary_genre_id
|
||||
|
||||
track_id = len(db.tracks) + 1
|
||||
# Reference exports always populate analyze_path even when the
|
||||
@@ -95,7 +225,29 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Datab
|
||||
bucket = f"P{(h >> 20) & 0x7F:02X}"
|
||||
analyze_id = f"{h & 0xFFFFFFFF:08X}"
|
||||
analyze_path = f"/PIONEER/USBANLZ/{bucket}/{analyze_id}/ANLZ0000.DAT"
|
||||
write_anlz(file_path, usb_root / analyze_path.lstrip("/"))
|
||||
anlz_abs = (usb_root / analyze_path.lstrip("/")).resolve()
|
||||
# `_analysis/` sits at the input collection root; ISRC-keyed cache
|
||||
# populated during `just collection`. write_anlz reads it before
|
||||
# falling back to a fresh ffmpeg+aubio pass.
|
||||
analysis = write_anlz(
|
||||
file_path, anlz_abs, source_audio=dest_abs,
|
||||
analysis_cache_dir=input_dir / "_analysis",
|
||||
)
|
||||
written.add(anlz_abs)
|
||||
# .EXT sister file is written automatically next to the .DAT.
|
||||
written.add(anlz_abs.with_suffix(".EXT"))
|
||||
# Prefer the analytically detected BPM over the file tag when we have one.
|
||||
# CDJ browse panel reads pdb.tempo; loaded-deck readout reads the PQTZ
|
||||
# we just wrote — keeping the two in sync avoids the "120.0 in browse,
|
||||
# blank on deck" UX bug.
|
||||
if analysis and analysis.get("bpm"):
|
||||
tempo_x100 = round(analysis["bpm"] * 100)
|
||||
# Also write the bpm TAG to the file (symmetric with `just
|
||||
# collection`'s _write_bpm_tag). overwrite=False so any
|
||||
# pre-existing label-set BPM is preserved.
|
||||
write_bpm_tag(dest_abs, analysis["bpm"], overwrite=False)
|
||||
else:
|
||||
tempo_x100 = meta.tempo
|
||||
db.tracks.append(Track(
|
||||
id=track_id,
|
||||
title=meta.title,
|
||||
@@ -110,7 +262,7 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Datab
|
||||
duration=meta.duration,
|
||||
track_number=meta.track_number,
|
||||
year=meta.year,
|
||||
tempo=meta.tempo,
|
||||
tempo=tempo_x100,
|
||||
date_added=today,
|
||||
analyze_path=analyze_path,
|
||||
analyze_date=today,
|
||||
@@ -120,9 +272,61 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Datab
|
||||
))
|
||||
track_by_src[original.resolve()] = track_id
|
||||
|
||||
# Cross-cutting validation: same checks as `just collection`. The
|
||||
# input audio_path is the staged file under data/usb/ so its tags
|
||||
# are what end up on the player. analysis cache lives at the source
|
||||
# collection root (input_dir/_analysis/<isrc>.json).
|
||||
track_issues = validate_track(
|
||||
dest_abs,
|
||||
analysis_cache_dir=input_dir / "_analysis",
|
||||
)
|
||||
validation_issues.extend(track_issues)
|
||||
|
||||
track_dt = time.perf_counter() - track_t0
|
||||
bpm_str = f"{analysis['bpm']:.1f} bpm" if (analysis and analysis.get("bpm")) else f"tag {tempo_x100/100:.1f} bpm"
|
||||
issue_tag = f" [{len(track_issues)} issue(s)]" if track_issues else ""
|
||||
logger.info(
|
||||
"[{}/{}] {} - {} ({}, {:.1f}s){}",
|
||||
track_idx, len(audio_files),
|
||||
meta.artist, meta.title,
|
||||
bpm_str, track_dt, issue_tag,
|
||||
)
|
||||
logger.info(
|
||||
"track table built in {:.1f}s ({} tracks, {} artists, {} albums, "
|
||||
"{} artwork, {} genres)",
|
||||
time.perf_counter() - build_t0,
|
||||
len(db.tracks), len(db.artists), len(db.albums),
|
||||
len(db.artwork), len(db.genres),
|
||||
)
|
||||
# Genre map summary: largest buckets first; helps spot mistagged tracks.
|
||||
if genre_to_tracks:
|
||||
logger.info("genre→tracks map ({} genres):", len(genre_to_tracks))
|
||||
for genre, track_ids in sorted(
|
||||
genre_to_tracks.items(), key=lambda kv: (-len(kv[1]), kv[0])
|
||||
):
|
||||
logger.info(" {:<32s} {:>3} track(s)", genre[:32], len(track_ids))
|
||||
no_genre = sum(1 for t in db.tracks if t.genre_id == 0)
|
||||
if no_genre:
|
||||
logger.warning("{} of {} tracks have no genre tag", no_genre, len(db.tracks))
|
||||
|
||||
# Emit the validation report. Same TrackIssue shape as `just collection`,
|
||||
# surfaced through loguru so the user gets visibility plus a roll-up at
|
||||
# the end.
|
||||
if validation_issues:
|
||||
logger.warning(
|
||||
"{} validation issue(s) across {} track(s)",
|
||||
len(validation_issues),
|
||||
len({(i.artist, i.title) for i in validation_issues}),
|
||||
)
|
||||
log_issues(validation_issues, report=False)
|
||||
else:
|
||||
logger.info("validation: all {} tracks passed", len(audio_files))
|
||||
|
||||
if not with_playlists:
|
||||
return db
|
||||
return db, written, validation_issues
|
||||
playlists = scan_playlists(input_dir)
|
||||
if playlists:
|
||||
logger.info("scanning {} playlist(s)", len(playlists))
|
||||
for sort_order, (name, paths) in enumerate(playlists.items(), start=1):
|
||||
playlist_id = len(db.playlist_nodes) + 1
|
||||
db.playlist_nodes.append(PlaylistNode(
|
||||
@@ -132,18 +336,58 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Datab
|
||||
name=name,
|
||||
is_folder=False,
|
||||
))
|
||||
resolved = 0
|
||||
for entry_index, p in enumerate(paths, start=1):
|
||||
tid = track_by_src.get(p.resolve())
|
||||
if tid is None:
|
||||
print(f"warning: playlist {name!r} references missing file {p}", file=sys.stderr)
|
||||
logger.warning("playlist {!r} references missing file {}", name, p)
|
||||
continue
|
||||
db.playlist_entries.append(PlaylistEntry(
|
||||
playlist_id=playlist_id,
|
||||
track_id=tid,
|
||||
entry_index=entry_index,
|
||||
))
|
||||
resolved += 1
|
||||
logger.info(" {}: {} entries ({} resolved, {} missing)",
|
||||
name, len(paths), resolved, len(paths) - resolved)
|
||||
|
||||
return db
|
||||
return db, written, validation_issues
|
||||
|
||||
|
||||
def prune_stale(usb_root: Path, kept: set[Path]) -> tuple[int, int]:
|
||||
"""Delete every file under `usb_root` not in `kept`, then drop empty dirs.
|
||||
|
||||
Returns (files_removed, dirs_removed). Operates only under `usb_root`,
|
||||
so anything outside is untouched. The `kept` set is expected to hold
|
||||
fully-resolved absolute paths.
|
||||
"""
|
||||
files_removed = 0
|
||||
for p in usb_root.rglob("*"):
|
||||
if not p.is_file():
|
||||
continue
|
||||
if p.resolve() in kept:
|
||||
continue
|
||||
try:
|
||||
p.unlink()
|
||||
files_removed += 1
|
||||
except OSError as e:
|
||||
logger.warning("failed to remove {}: {}", p, e)
|
||||
|
||||
# Bottom-up empty-dir removal: deepest first so each parent gets a
|
||||
# chance to become empty once its children are gone.
|
||||
dirs = sorted(
|
||||
(d for d in usb_root.rglob("*") if d.is_dir()),
|
||||
key=lambda d: len(d.parts),
|
||||
reverse=True,
|
||||
)
|
||||
dirs_removed = 0
|
||||
for d in dirs:
|
||||
try:
|
||||
d.rmdir()
|
||||
dirs_removed += 1
|
||||
except OSError:
|
||||
pass # not empty; expected for the directories we still want
|
||||
return files_removed, dirs_removed
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
@@ -151,16 +395,41 @@ def main(argv: list[str] | None = None) -> int:
|
||||
parser.add_argument("input_dir", nargs="?", type=Path, default=Path("./data/input"))
|
||||
parser.add_argument("output_root", nargs="?", type=Path, default=Path("./data/usb"))
|
||||
parser.add_argument("--no-playlists", action="store_true", help="Skip m3u8 playlist scanning")
|
||||
parser.add_argument("--no-prune", action="store_true", help="Skip the post-build sweep of stale files")
|
||||
parser.add_argument("--strict", action="store_true", help="Fail with non-zero exit if any track fails validation")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
configure_logging()
|
||||
if not args.input_dir.is_dir():
|
||||
parser.error(f"input_dir {args.input_dir} is not a directory")
|
||||
|
||||
db = build(args.input_dir, args.output_root, with_playlists=not args.no_playlists)
|
||||
overall_t0 = time.perf_counter()
|
||||
logger.info("export: {} → {}", args.input_dir, args.output_root)
|
||||
db, written, validation_issues = build(args.input_dir, args.output_root, with_playlists=not args.no_playlists)
|
||||
|
||||
pdb_path = args.output_root / "PIONEER" / "rekordbox" / "export.pdb"
|
||||
write_pdb(db, pdb_path)
|
||||
written.add(pdb_path.resolve())
|
||||
|
||||
print(f"wrote {len(db.tracks)} tracks, {len(db.artists)} artists, {len(db.albums)} albums, "
|
||||
f"{len(db.playlist_nodes)} playlists ({len(db.playlist_entries)} entries) to {pdb_path}")
|
||||
if not args.no_prune:
|
||||
t = time.perf_counter()
|
||||
n_files, n_dirs = prune_stale(args.output_root, written)
|
||||
if n_files or n_dirs:
|
||||
logger.info(
|
||||
"pruned {} stale file(s) + {} empty dir(s) in {:.1f}s",
|
||||
n_files, n_dirs, time.perf_counter() - t,
|
||||
)
|
||||
else:
|
||||
logger.debug("nothing to prune ({:.1f}s)", time.perf_counter() - t)
|
||||
|
||||
logger.info(
|
||||
"done in {:.1f}s: {} tracks, {} artists, {} albums, "
|
||||
"{} playlists ({} entries) → {}",
|
||||
time.perf_counter() - overall_t0,
|
||||
len(db.tracks), len(db.artists), len(db.albums),
|
||||
len(db.playlist_nodes), len(db.playlist_entries), pdb_path,
|
||||
)
|
||||
if args.strict and validation_issues:
|
||||
logger.error("--strict: refusing to succeed with {} validation issue(s)", len(validation_issues))
|
||||
return 1
|
||||
return 0
|
||||
@@ -21,7 +21,18 @@ Env (from .env, loaded by just):
|
||||
MAPLE_SPOTIFY_CLIENT_ID, MAPLE_SPOTIFY_CLIENT_SECRET (spotify.py reads)
|
||||
SOCKSEEK_USER, SOCKSEEK_PASS (forwarded to sockseek)
|
||||
"""
|
||||
from maple.collection.core import Collection, TrackIssue
|
||||
from maple.collection.index import CollectionIndex
|
||||
|
||||
# Re-exports are lazy via __getattr__ so importing `maple.collection.index`
|
||||
# (or any other submodule) doesn't drag the whole package — `maple.validate`
|
||||
# depends on `maple.collection.index` and would otherwise circular-import
|
||||
# through `maple.collection.core` → `maple.validate` → here.
|
||||
__all__ = ["Collection", "CollectionIndex", "TrackIssue"]
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
if name == "Collection" or name == "TrackIssue":
|
||||
from maple.collection.core import Collection, TrackIssue
|
||||
return {"Collection": Collection, "TrackIssue": TrackIssue}[name]
|
||||
if name == "CollectionIndex":
|
||||
from maple.collection.index import CollectionIndex
|
||||
return CollectionIndex
|
||||
raise AttributeError(name)
|
||||
@@ -16,6 +16,7 @@ from loguru import logger
|
||||
|
||||
from maple.collection.core import Collection, TrackIssue
|
||||
from maple.collection.index import first_artist, norm
|
||||
from maple.enrich import save_spotify_manifest
|
||||
from maple.log import configure as configure_logging
|
||||
from maple.spotify import fetch_playlists
|
||||
|
||||
@@ -55,6 +56,9 @@ async def main_async(argv: list[str]) -> int:
|
||||
playlists = await fetch_playlists(args.ids)
|
||||
total = sum(len(p["tracks"]) for p in playlists)
|
||||
logger.info("fetched {} playlist(s), {} tracks", len(playlists), total)
|
||||
# Persist the full set of Spotify tracks so `just export` (and any
|
||||
# later `just collection` run) can enrich tags without re-fetching.
|
||||
save_spotify_manifest(collection.metadata_dir, playlists)
|
||||
|
||||
collection.refresh_index()
|
||||
logger.info(
|
||||
|
||||
@@ -23,6 +23,10 @@ import aiohttp
|
||||
from loguru import logger
|
||||
from mutagen import File as MutagenFile
|
||||
|
||||
from maple.analyze import analyze_audio, detect_bpm, save_analysis
|
||||
from maple.enrich import write_spotify_tags as _shared_write_spotify_tags
|
||||
from maple.validate import TrackIssue as _ValidateTrackIssue
|
||||
from maple.validate import validate_track as _validate_track
|
||||
from maple.collection.index import (
|
||||
CollectionIndex,
|
||||
first_artist,
|
||||
@@ -42,13 +46,14 @@ from maple.collection.validate import (
|
||||
)
|
||||
from maple.scan import AUDIO_EXTS
|
||||
|
||||
BPM_DIVERGENCE_WARN = 0.05 # warn when aubio vs existing tag differs >5%
|
||||
|
||||
@dataclass
|
||||
class TrackIssue:
|
||||
playlist: str
|
||||
artist: str
|
||||
title: str
|
||||
reason: str
|
||||
|
||||
# Re-export the shared validator's TrackIssue so callers that imported it
|
||||
# from maple.collection.core continue to work. There is one TrackIssue
|
||||
# definition in the codebase — `maple.validate.TrackIssue` — and both the
|
||||
# `just collection` and `just export` flows produce/consume the same shape.
|
||||
TrackIssue = _ValidateTrackIssue
|
||||
|
||||
|
||||
_M3U_BAD = re.compile(r"[\\/\x00]+")
|
||||
@@ -95,10 +100,17 @@ class Collection:
|
||||
def report_path(self) -> Path:
|
||||
return self.metadata_dir / "collection-report.txt"
|
||||
|
||||
@property
|
||||
def analysis_dir(self) -> Path:
|
||||
"""`<root>/_analysis/<isrc>.json` — cached aubio output keyed on ISRC,
|
||||
consumed by `just export` to skip the decode+detect pass."""
|
||||
return self.root / "_analysis"
|
||||
|
||||
def ensure_dirs(self) -> None:
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
self.metadata_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.playlists_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.analysis_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# --- startup validation ----------------------------------------------- #
|
||||
|
||||
@@ -405,58 +417,38 @@ class Collection:
|
||||
return issues
|
||||
|
||||
def write_spotify_tags(self, track: dict, path: Path) -> None:
|
||||
"""Overwrite the file's tags with the Spotify values.
|
||||
"""Delegate to the shared `maple.enrich.write_spotify_tags` so the
|
||||
collection-side and export-side writes are byte-for-byte identical
|
||||
and can't drift."""
|
||||
_shared_write_spotify_tags(track, path)
|
||||
|
||||
Spotify is canonical: title, artist (joined ", "), album, date,
|
||||
genre(s), and ISRC are written. Missing Spotify values leave the
|
||||
existing tag untouched.
|
||||
"""
|
||||
def _existing_bpm(self, path: Path) -> float | None:
|
||||
"""Return the parsed `bpm` tag value, or None if absent/invalid."""
|
||||
try:
|
||||
audio = MutagenFile(path, easy=True)
|
||||
if audio is None or audio.tags is None:
|
||||
return None
|
||||
v = audio.tags.get("bpm")
|
||||
if not v:
|
||||
return None
|
||||
return float(v[0])
|
||||
except (ValueError, TypeError, Exception):
|
||||
return None
|
||||
|
||||
def _write_bpm_tag(self, path: Path, bpm: float) -> None:
|
||||
try:
|
||||
audio = MutagenFile(path, easy=True)
|
||||
except Exception as e:
|
||||
logger.warning("write_spotify_tags: cannot open {}: {}", path.name, e)
|
||||
return
|
||||
if audio is None:
|
||||
return
|
||||
if audio.tags is None:
|
||||
try:
|
||||
audio.add_tags()
|
||||
except Exception as e:
|
||||
logger.warning("write_spotify_tags: cannot init tags on {}: {}", path.name, e)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
title = (track.get("title") or "").strip()
|
||||
if title:
|
||||
audio.tags["title"] = title
|
||||
artists = track.get("artists") or []
|
||||
if artists:
|
||||
audio.tags["artist"] = ", ".join(artists)
|
||||
album = (track.get("album") or "").strip()
|
||||
if album:
|
||||
audio.tags["album"] = album
|
||||
date = (track.get("release_date") or "").strip()
|
||||
if date:
|
||||
try:
|
||||
audio.tags["date"] = date
|
||||
except (KeyError, ValueError) as e:
|
||||
logger.debug("date tag write skipped on {}: {}", path.name, e)
|
||||
genres = track.get("genres") or []
|
||||
if genres:
|
||||
try:
|
||||
audio.tags["genre"] = list(genres)
|
||||
except (KeyError, ValueError) as e:
|
||||
logger.debug("genre tag write skipped on {}: {}", path.name, e)
|
||||
isrc = (track.get("isrc") or "").strip()
|
||||
if isrc:
|
||||
try:
|
||||
audio.tags["isrc"] = isrc
|
||||
except (KeyError, ValueError) as e:
|
||||
logger.debug("isrc tag write skipped on {}: {}", path.name, e)
|
||||
|
||||
try:
|
||||
audio.tags["bpm"] = f"{bpm:.2f}"
|
||||
audio.save()
|
||||
except Exception as e:
|
||||
logger.warning("write_spotify_tags: save failed for {}: {}", path.name, e)
|
||||
logger.warning("failed to write bpm to {}: {}", path.name, e)
|
||||
|
||||
def process_track(
|
||||
self,
|
||||
@@ -464,12 +456,14 @@ class Collection:
|
||||
track: dict,
|
||||
path: Path,
|
||||
) -> list[TrackIssue]:
|
||||
"""Reconcile a single downloaded file against Spotify.
|
||||
"""Reconcile a single downloaded file against Spotify + analyse it.
|
||||
|
||||
Steps:
|
||||
1. ffprobe sanity check (one issue + early return on failure).
|
||||
2. Compare every relevant field; collect mismatch issues.
|
||||
3. Write Spotify's values back into the file's tags.
|
||||
4. If the file has no `bpm` tag yet, run aubio BPM detection
|
||||
and write the analytical value. Existing tags are left alone.
|
||||
|
||||
Idempotent within a run — paths already processed are no-ops.
|
||||
"""
|
||||
@@ -477,14 +471,49 @@ class Collection:
|
||||
return []
|
||||
self._reconciled.add(path)
|
||||
|
||||
sp_artist = ", ".join(track.get("artists") or [])
|
||||
sp_title = track.get("title") or ""
|
||||
if not ffprobe_ok(path):
|
||||
return [TrackIssue(playlist_name, sp_artist, sp_title, f"ffprobe failed: {path}")]
|
||||
|
||||
issues = self.compare_against_spotify(playlist_name, track, path)
|
||||
# Source-of-truth tag writes first: ensures the file mirrors Spotify
|
||||
# before validation reads its tags back. Mismatches caught here are
|
||||
# written to the report regardless of the validator's subsequent
|
||||
# findings (the validator may dedup them if Spotify still disagrees).
|
||||
self.write_spotify_tags(track, path)
|
||||
return issues
|
||||
|
||||
# Single audio analysis pass: BPM + beat grid + waveform envelopes.
|
||||
# Cached under collection/_analysis/<isrc>.json so `just export`
|
||||
# reads the bytes back instead of redoing ffmpeg+aubio.
|
||||
isrc = (track.get("isrc") or "").strip()
|
||||
existing_bpm = self._existing_bpm(path)
|
||||
try:
|
||||
analysis = analyze_audio(path)
|
||||
except Exception as e:
|
||||
logger.warning("analyze_audio failed on {}: {}", path.name, e)
|
||||
analysis = None
|
||||
if analysis and analysis.get("bpm"):
|
||||
if existing_bpm is None:
|
||||
self._write_bpm_tag(path, analysis["bpm"])
|
||||
logger.info("{}: bpm {:.2f} (aubio)", path.name, analysis["bpm"])
|
||||
else:
|
||||
logger.debug(
|
||||
"{}: bpm {:.2f} (existing tag, kept; aubio measured {:.2f})",
|
||||
path.name, existing_bpm, analysis["bpm"],
|
||||
)
|
||||
if isrc:
|
||||
cp = save_analysis(self.analysis_dir, isrc, analysis)
|
||||
logger.info("cached analysis for {}: {} → {}", isrc, path.name, cp)
|
||||
else:
|
||||
logger.warning(
|
||||
"skipping analysis cache for {}: no ISRC in spotify track ({} - {})",
|
||||
path.name, ", ".join(track.get("artists") or []), track.get("title") or "?",
|
||||
)
|
||||
|
||||
# Full cross-cutting validation pass. Runs the SAME checks as
|
||||
# `just export`, so a clean run here means the track will export
|
||||
# without issues — and vice versa.
|
||||
return _validate_track(
|
||||
path,
|
||||
spotify_track=track,
|
||||
analysis_cache_dir=self.analysis_dir,
|
||||
playlist=playlist_name,
|
||||
)
|
||||
|
||||
# --- validation -------------------------------------------------------- #
|
||||
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
"""Spotify metadata enrichment shared by `just collection` and `just export`.
|
||||
|
||||
Per the project convention: a track is "valid" iff every export criterion
|
||||
holds (artwork, tags, analysis). BOTH pipelines must be capable of
|
||||
*bringing* a track to that state — validation isn't just reporting,
|
||||
it's repair.
|
||||
|
||||
`just collection`:
|
||||
1. Fetches Spotify playlists.
|
||||
2. Persists every track to `<collection>/_metadata/spotify-tracks.json`.
|
||||
3. Walks the collection, calls `write_spotify_tags` for each match.
|
||||
|
||||
`just export`:
|
||||
1. Loads the manifest written by `just collection`.
|
||||
2. Walks `data/usb/`'s staged files. For any file missing canonical
|
||||
tags (typically ISRC), looks up its Spotify entry via the manifest
|
||||
and calls `write_spotify_tags` BEFORE validation runs.
|
||||
|
||||
The shared write function lives here so the two pipelines are
|
||||
byte-for-byte consistent in what they leave on disk.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from loguru import logger
|
||||
from mutagen import File as MutagenFile
|
||||
from mutagen.flac import FLAC, Picture
|
||||
from mutagen.id3 import APIC, ID3, ID3NoHeaderError
|
||||
from mutagen.mp4 import MP4, MP4Cover
|
||||
|
||||
from maple.collection.index import first_artist, norm
|
||||
from maple.collection.validate import embed_art, has_embedded_art
|
||||
|
||||
MANIFEST_NAME = "spotify-tracks.json"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Manifest persistence #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def save_spotify_manifest(metadata_dir: Path, playlists: list[dict]) -> Path:
|
||||
"""Persist Spotify-fetched playlists so `just export` can read them later.
|
||||
|
||||
Stored as a single JSON blob at `<metadata_dir>/spotify-tracks.json`.
|
||||
Replaces any prior manifest atomically (tmp + rename).
|
||||
"""
|
||||
metadata_dir.mkdir(parents=True, exist_ok=True)
|
||||
target = metadata_dir / MANIFEST_NAME
|
||||
tmp = target.with_suffix(target.suffix + ".tmp")
|
||||
tmp.write_text(
|
||||
json.dumps({"playlists": playlists}, indent=2, ensure_ascii=False, default=str)
|
||||
)
|
||||
tmp.replace(target)
|
||||
n = sum(len(p.get("tracks") or []) for p in playlists)
|
||||
logger.info("wrote spotify manifest: {} tracks across {} playlist(s) → {}",
|
||||
n, len(playlists), target)
|
||||
return target
|
||||
|
||||
|
||||
def load_spotify_manifest(
|
||||
metadata_dir: Path,
|
||||
) -> tuple[dict[tuple[str, str], dict], dict[str, dict]]:
|
||||
"""Load the manifest written by `just collection`.
|
||||
|
||||
Returns `(by_key, by_isrc)` where `by_key` is indexed by
|
||||
`(first_artist_norm, title_norm)` and `by_isrc` is indexed by the
|
||||
uppercased ISRC. Empty dicts if the manifest is absent/unreadable.
|
||||
"""
|
||||
target = metadata_dir / MANIFEST_NAME
|
||||
if not target.exists():
|
||||
return {}, {}
|
||||
try:
|
||||
data = json.loads(target.read_text())
|
||||
except Exception as e:
|
||||
logger.warning("could not read spotify manifest at {}: {}", target, e)
|
||||
return {}, {}
|
||||
by_key: dict[tuple[str, str], dict] = {}
|
||||
by_isrc: dict[str, dict] = {}
|
||||
for pl in data.get("playlists", []):
|
||||
for t in pl.get("tracks", []):
|
||||
artist = ", ".join(t.get("artists") or [])
|
||||
k = (norm(first_artist(artist)), norm(t.get("title") or ""))
|
||||
by_key.setdefault(k, t)
|
||||
isrc = (t.get("isrc") or "").strip().upper()
|
||||
if isrc:
|
||||
by_isrc.setdefault(isrc, t)
|
||||
return by_key, by_isrc
|
||||
|
||||
|
||||
def find_spotify_track(
|
||||
path: Path,
|
||||
by_key: dict[tuple[str, str], dict],
|
||||
by_isrc: dict[str, dict],
|
||||
) -> dict | None:
|
||||
"""Resolve a file to its Spotify track via ISRC tag, then (artist, title)."""
|
||||
try:
|
||||
audio = MutagenFile(path, easy=True)
|
||||
except Exception:
|
||||
return None
|
||||
if audio is None or audio.tags is None:
|
||||
return None
|
||||
tags = audio.tags
|
||||
|
||||
def first(k: str) -> str:
|
||||
if hasattr(tags, "get"):
|
||||
v = tags.get(k)
|
||||
if v:
|
||||
return str(v[0])
|
||||
return ""
|
||||
|
||||
isrc = first("isrc").upper().strip()
|
||||
if isrc and isrc in by_isrc:
|
||||
return by_isrc[isrc]
|
||||
artist = first("artist")
|
||||
title = first("title")
|
||||
if artist or title:
|
||||
k = (norm(first_artist(artist)), norm(title))
|
||||
return by_key.get(k)
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Tag writing (the canonical write path) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def write_spotify_tags(track: dict, path: Path) -> bool:
|
||||
"""Overwrite the file's tags with the Spotify-canonical values.
|
||||
|
||||
Both pipelines call this. Writes title, artist (`", ".join`),
|
||||
album, date, genre (list), and ISRC via mutagen's easy interface.
|
||||
Empty Spotify values leave the existing tag untouched.
|
||||
|
||||
Returns True when at least one tag was written + saved.
|
||||
"""
|
||||
try:
|
||||
audio = MutagenFile(path, easy=True)
|
||||
except Exception as e:
|
||||
logger.warning("write_spotify_tags: cannot open {}: {}", path.name, e)
|
||||
return False
|
||||
if audio is None:
|
||||
return False
|
||||
if audio.tags is None:
|
||||
try:
|
||||
audio.add_tags()
|
||||
except Exception as e:
|
||||
logger.warning("write_spotify_tags: cannot init tags on {}: {}", path.name, e)
|
||||
return False
|
||||
|
||||
wrote_any = False
|
||||
title = (track.get("title") or "").strip()
|
||||
if title:
|
||||
audio.tags["title"] = title
|
||||
wrote_any = True
|
||||
artists = track.get("artists") or []
|
||||
if artists:
|
||||
audio.tags["artist"] = ", ".join(artists)
|
||||
wrote_any = True
|
||||
album = (track.get("album") or "").strip()
|
||||
if album:
|
||||
audio.tags["album"] = album
|
||||
wrote_any = True
|
||||
date = (track.get("release_date") or "").strip()
|
||||
if date:
|
||||
try:
|
||||
audio.tags["date"] = date
|
||||
wrote_any = True
|
||||
except (KeyError, ValueError) as e:
|
||||
logger.debug("date tag write skipped on {}: {}", path.name, e)
|
||||
genres = track.get("genres") or []
|
||||
if genres:
|
||||
try:
|
||||
audio.tags["genre"] = list(genres)
|
||||
wrote_any = True
|
||||
except (KeyError, ValueError) as e:
|
||||
logger.debug("genre tag write skipped on {}: {}", path.name, e)
|
||||
isrc = (track.get("isrc") or "").strip()
|
||||
if isrc:
|
||||
try:
|
||||
audio.tags["isrc"] = isrc
|
||||
wrote_any = True
|
||||
except (KeyError, ValueError) as e:
|
||||
logger.debug("isrc tag write skipped on {}: {}", path.name, e)
|
||||
|
||||
if not wrote_any:
|
||||
return False
|
||||
try:
|
||||
audio.save()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning("write_spotify_tags: save failed for {}: {}", path.name, e)
|
||||
return False
|
||||
|
||||
|
||||
def write_bpm_tag(path: Path, bpm: float, *, overwrite: bool = False) -> bool:
|
||||
"""Write a `bpm` tag to the file. When `overwrite=False` (the default),
|
||||
leaves any existing tag value alone — preserves human/label-set BPMs."""
|
||||
try:
|
||||
audio = MutagenFile(path, easy=True)
|
||||
except Exception as e:
|
||||
logger.warning("write_bpm_tag: cannot open {}: {}", path.name, e)
|
||||
return False
|
||||
if audio is None:
|
||||
return False
|
||||
if audio.tags is None:
|
||||
try:
|
||||
audio.add_tags()
|
||||
except Exception as e:
|
||||
logger.warning("write_bpm_tag: cannot init tags on {}: {}", path.name, e)
|
||||
return False
|
||||
if not overwrite:
|
||||
existing = audio.tags.get("bpm") if hasattr(audio.tags, "get") else None
|
||||
if existing and (existing[0] if isinstance(existing, list) else existing):
|
||||
return False # keep the existing tag
|
||||
audio.tags["bpm"] = f"{bpm:.2f}"
|
||||
try:
|
||||
audio.save()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning("write_bpm_tag: save failed for {}: {}", path.name, e)
|
||||
return False
|
||||
|
||||
|
||||
def download_jpeg(url: str, *, timeout: float = 15.0) -> bytes | None:
|
||||
"""Sync GET of a Spotify album_image_url. Returns the raw bytes or
|
||||
None on failure. Used by `embed_artwork_from_track` — kept sync so
|
||||
both `just collection`'s async loop and `just export`'s sync loop
|
||||
can call into the same helper without contortions."""
|
||||
if not url:
|
||||
return None
|
||||
try:
|
||||
import urllib.request
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "maple/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.read()
|
||||
except Exception as e:
|
||||
logger.warning("artwork fetch failed for {}: {}", url, e)
|
||||
return None
|
||||
|
||||
|
||||
def embed_artwork_from_track(track: dict, path: Path, *, force: bool = False) -> bool:
|
||||
"""Fetch the Spotify cover art for `track` and embed it into `path`.
|
||||
|
||||
No-op when the file already has embedded art (unless `force=True`).
|
||||
Skips silently when the track dict has no `album_image_url` (Spotify
|
||||
occasionally returns tracks without images).
|
||||
"""
|
||||
if not force and has_embedded_art(path):
|
||||
return False
|
||||
url = (track.get("album_image_url") or "").strip()
|
||||
if not url:
|
||||
return False
|
||||
data = download_jpeg(url)
|
||||
if not data:
|
||||
return False
|
||||
try:
|
||||
embed_art(path, data)
|
||||
logger.info(
|
||||
"embedded artwork for {} (from {})",
|
||||
path.name, url.rsplit("/", 1)[-1][:40],
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning("embed_art failed for {}: {}", path.name, e)
|
||||
return False
|
||||
|
||||
|
||||
def enrich_from_manifest(
|
||||
path: Path,
|
||||
by_key: dict[tuple[str, str], dict],
|
||||
by_isrc: dict[str, dict],
|
||||
analysis_cache_dir: Path | None = None,
|
||||
) -> dict | None:
|
||||
"""Find this file's Spotify entry, bring it to "valid" state, return
|
||||
the matched track dict (or None if no match).
|
||||
|
||||
Steps:
|
||||
1. Write canonical text tags via `write_spotify_tags`.
|
||||
2. Fetch + embed album art if the file lacks it (`embed_artwork_from_track`).
|
||||
3. If `analysis_cache_dir` is supplied and the cache has an entry for
|
||||
this track's ISRC, write the cached BPM as a tag. This lets the
|
||||
enrich pass bring a track all the way to "validates" without a
|
||||
separate aubio-analysis step.
|
||||
"""
|
||||
track = find_spotify_track(path, by_key, by_isrc)
|
||||
if track is None:
|
||||
return None
|
||||
if write_spotify_tags(track, path):
|
||||
logger.info(
|
||||
"enriched {} from spotify (isrc={!r}, title={!r})",
|
||||
path.name, (track.get("isrc") or "")[:12], (track.get("title") or "")[:32],
|
||||
)
|
||||
embed_artwork_from_track(track, path)
|
||||
|
||||
# Backfill BPM tag from the analysis cache when present. Avoids
|
||||
# re-running aubio here — if the cache is cold, the per-track loop's
|
||||
# write_anlz will populate it and write the tag in a later pass.
|
||||
isrc = (track.get("isrc") or "").strip()
|
||||
if analysis_cache_dir is not None and isrc:
|
||||
from maple.analyze import load_analysis
|
||||
cached = load_analysis(analysis_cache_dir, isrc)
|
||||
if cached and cached.get("bpm"):
|
||||
write_bpm_tag(path, cached["bpm"], overwrite=False)
|
||||
return track
|
||||
@@ -2,6 +2,8 @@
|
||||
from pathlib import Path
|
||||
from struct import pack, unpack_from
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from maple.model import (
|
||||
DEFAULT_COLUMNS,
|
||||
STANDARD_COLORS,
|
||||
@@ -98,11 +100,21 @@ def _next_sequence(path: Path) -> int:
|
||||
|
||||
|
||||
def write_pdb(db: Database, path: str | Path) -> None:
|
||||
logger.info(
|
||||
"building pdb → {} (tracks={} artists={} albums={} genres={} "
|
||||
"artwork={} playlists={} entries={})",
|
||||
path,
|
||||
len(db.tracks), len(db.artists), len(db.albums), len(db.genres),
|
||||
len(db.artwork), len(db.playlist_nodes), len(db.playlist_entries),
|
||||
)
|
||||
# Refuse to emit a pdb with dangling foreign keys -- a track pointing at a
|
||||
# genre/artist/album/artwork row that doesn't exist makes the player follow
|
||||
# the FK into an empty table and freeze (freeze-class, like the deck hang).
|
||||
fk_errors = validate_referential_integrity(db)
|
||||
if fk_errors:
|
||||
logger.error("FK validation failed with {} error(s)", len(fk_errors))
|
||||
for e in fk_errors:
|
||||
logger.error(" {}", e)
|
||||
raise ValueError("dangling foreign keys (would freeze the player):\n " + "\n ".join(fk_errors))
|
||||
|
||||
# Layout (matching rekordbox): page 0 is the file header. Pages 1..2*N are
|
||||
@@ -215,6 +227,7 @@ def write_pdb(db: Database, path: str | Path) -> None:
|
||||
b.next_page = free_sentinel
|
||||
for b in last_builders:
|
||||
pages_by_index[b.page_index] = b.finalize()
|
||||
overflow_used = overflow_next - (1 + 2 * num_tables)
|
||||
|
||||
# Materialise a small free pool of zero-filled pages and set next_unused PAST
|
||||
# it, mirroring a real export (e.g. data/MAPLE: 71 pages, next_unused=74 ->
|
||||
@@ -232,6 +245,10 @@ def write_pdb(db: Database, path: str | Path) -> None:
|
||||
next_unused_page = file_page_count + 2
|
||||
|
||||
sequence = _next_sequence(Path(path))
|
||||
logger.info(
|
||||
"pdb layout: {} pages ({} reserved, {} overflow, {} free), sequence={}",
|
||||
file_page_count, 2 * num_tables, overflow_used, FREE_POOL_PAGES, sequence,
|
||||
)
|
||||
|
||||
file_header = b""
|
||||
file_header += pack("<I", 0) # magic
|
||||
@@ -268,3 +285,4 @@ def write_pdb(db: Database, path: str | Path) -> None:
|
||||
page = pages_by_index.get(idx, blank)
|
||||
assert len(page) == PAGE_SIZE
|
||||
f.write(page)
|
||||
logger.info("wrote {} ({} bytes)", out_path, out_path.stat().st_size)
|
||||
@@ -0,0 +1,261 @@
|
||||
"""Canonical `Track` — the unit shared by `just collection` and `just export`.
|
||||
|
||||
A *collection* is a sequence of `Track`s persisted under
|
||||
`data/collection/`. An *export* reads a sequence of `Track`s and writes a
|
||||
rekordbox USB at `data/usb/`. This dataclass is the bridge: any code that
|
||||
talks about "a track" in maple should hold one of these.
|
||||
|
||||
Field semantics:
|
||||
- Identity-ish fields (`isrc`, `title`, `artists`, `album`) start populated
|
||||
from whichever source first knows the track — Spotify, a file's existing
|
||||
tags, or a manual entry.
|
||||
- Spotify-only fields (`spotify_track_id`, `release_date`, …) stay empty
|
||||
for tracks that didn't come through the Spotify path.
|
||||
- On-disk fields (`path`, `bitrate`, …) are populated once the audio is
|
||||
actually on disk somewhere.
|
||||
- Analytical fields (`bpm`, `beats`, …) populate during the audio-analysis
|
||||
pass and are cached under `<collection>/_analysis/<isrc>.json`.
|
||||
|
||||
The existing `maple.model.Track` is a *denormalised pdb row* — it carries
|
||||
artist_id/album_id/genre_id FKs that only exist in the context of a
|
||||
specific export. `Track.to_pdb_row(...)` projects this canonical Track
|
||||
into that shape using caller-supplied FK indices.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from maple import model
|
||||
|
||||
|
||||
@dataclass
|
||||
class Track:
|
||||
"""Canonical track. Defaults intentionally permissive so partially-filled
|
||||
instances flow through the pipeline; validation (`maple.validate`)
|
||||
decides what counts as "complete enough to export"."""
|
||||
|
||||
# ---- identity --------------------------------------------------------- #
|
||||
isrc: str = ""
|
||||
title: str = ""
|
||||
artists: list[str] = field(default_factory=list)
|
||||
album: str = ""
|
||||
|
||||
# ---- Spotify-derived ------------------------------------------------- #
|
||||
spotify_track_id: str = ""
|
||||
spotify_artist_ids: list[str] = field(default_factory=list)
|
||||
album_image_url: str = ""
|
||||
release_date: str = "" # "YYYY", "YYYY-MM", or "YYYY-MM-DD"
|
||||
release_date_precision: str = ""
|
||||
genres: list[str] = field(default_factory=list)
|
||||
track_number: int = 0
|
||||
disc_number: int = 0
|
||||
duration_ms: int = 0
|
||||
|
||||
# ---- on-disk audio --------------------------------------------------- #
|
||||
path: Path | None = None # current best file location
|
||||
file_size: int = 0
|
||||
bitrate: int = 0 # kbps
|
||||
sample_rate: int = 0 # Hz
|
||||
sample_depth: int = 0 # bits per sample
|
||||
|
||||
# ---- analytical ------------------------------------------------------ #
|
||||
bpm: float | None = None # median-of-intervals tempo
|
||||
|
||||
# ---- provenance ------------------------------------------------------ #
|
||||
source: str = "" # "spotify", "local", "bandcamp", …
|
||||
|
||||
# ---- views ----------------------------------------------------------- #
|
||||
|
||||
@property
|
||||
def primary_artist(self) -> str:
|
||||
return self.artists[0] if self.artists else ""
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
if not self.title:
|
||||
return self.path.name if self.path else "<unknown>"
|
||||
if self.artists:
|
||||
return f"{', '.join(self.artists)} - {self.title}"
|
||||
return self.title
|
||||
|
||||
@property
|
||||
def primary_genre(self) -> str:
|
||||
return self.genres[0] if self.genres else ""
|
||||
|
||||
@property
|
||||
def year(self) -> int:
|
||||
"""First 4 digits of release_date as an int (0 if absent/invalid)."""
|
||||
if not self.release_date or len(self.release_date) < 4:
|
||||
return 0
|
||||
try:
|
||||
return int(self.release_date[:4])
|
||||
except ValueError:
|
||||
return 0
|
||||
|
||||
@property
|
||||
def duration_s(self) -> int:
|
||||
return int(round(self.duration_ms / 1000)) if self.duration_ms else 0
|
||||
|
||||
# ---- ingress --------------------------------------------------------- #
|
||||
|
||||
@classmethod
|
||||
def from_spotify(cls, d: dict) -> "Track":
|
||||
"""Build from one row of `maple.spotify.fetch_playlist`'s output."""
|
||||
return cls(
|
||||
isrc=(d.get("isrc") or "").strip(),
|
||||
title=d.get("title") or "",
|
||||
artists=list(d.get("artists") or []),
|
||||
album=d.get("album") or "",
|
||||
spotify_track_id=d.get("id") or "",
|
||||
spotify_artist_ids=list(d.get("artist_ids") or []),
|
||||
album_image_url=d.get("album_image_url") or "",
|
||||
release_date=d.get("release_date") or "",
|
||||
release_date_precision=d.get("release_date_precision") or "",
|
||||
genres=list(d.get("genres") or []),
|
||||
track_number=int(d.get("track_number") or 0),
|
||||
disc_number=int(d.get("disc_number") or 0),
|
||||
duration_ms=int(d.get("duration_ms") or 0),
|
||||
source="spotify",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: Path) -> "Track":
|
||||
"""Construct from on-disk audio via mutagen. Fields not in the file's
|
||||
tags stay at their defaults; the caller can `merge_spotify` later to
|
||||
fill them in canonically."""
|
||||
from mutagen import File as MutagenFile
|
||||
|
||||
out = cls(path=path, source="local")
|
||||
try:
|
||||
audio = MutagenFile(path, easy=True)
|
||||
except Exception:
|
||||
return out
|
||||
if audio is None:
|
||||
return out
|
||||
tags = audio.tags or {}
|
||||
def first(k: str) -> str:
|
||||
v = tags.get(k) if hasattr(tags, "get") else None
|
||||
return (v[0] if v else "") if isinstance(v, list) else (v or "")
|
||||
out.title = first("title")
|
||||
artist = first("artist")
|
||||
if artist:
|
||||
out.artists = [a.strip() for a in artist.split(",") if a.strip()]
|
||||
out.album = first("album")
|
||||
out.isrc = first("isrc").strip()
|
||||
out.release_date = first("date")
|
||||
gtag = first("genre")
|
||||
if gtag:
|
||||
from maple.cli import split_genre_tags
|
||||
out.genres = split_genre_tags(gtag)
|
||||
try:
|
||||
out.bpm = float(first("bpm")) if first("bpm") else None
|
||||
except ValueError:
|
||||
out.bpm = None
|
||||
try:
|
||||
out.track_number = int(first("tracknumber").split("/")[0])
|
||||
except ValueError:
|
||||
out.track_number = 0
|
||||
info = getattr(audio, "info", None)
|
||||
if info is not None:
|
||||
out.duration_ms = int(round((getattr(info, "length", 0.0) or 0.0) * 1000))
|
||||
bps = int(getattr(info, "bitrate", 0) or 0)
|
||||
out.bitrate = bps // 1000 if bps > 1000 else bps
|
||||
out.sample_rate = int(getattr(info, "sample_rate", 0) or 0)
|
||||
out.sample_depth = int(getattr(info, "bits_per_sample", 0) or 0)
|
||||
try:
|
||||
out.file_size = path.stat().st_size
|
||||
except OSError:
|
||||
pass
|
||||
return out
|
||||
|
||||
# ---- enrichment ------------------------------------------------------ #
|
||||
|
||||
def merge_spotify(self, d: dict) -> "Track":
|
||||
"""Overlay Spotify-canonical fields onto this Track *in place* — used
|
||||
when we have a local file but Spotify carries better identity / genre
|
||||
/ release date. Empty Spotify fields are NOT applied (no clobbering)."""
|
||||
if d.get("title"):
|
||||
self.title = d["title"]
|
||||
if d.get("artists"):
|
||||
self.artists = list(d["artists"])
|
||||
if d.get("album"):
|
||||
self.album = d["album"]
|
||||
if d.get("isrc"):
|
||||
self.isrc = d["isrc"].strip()
|
||||
if d.get("id"):
|
||||
self.spotify_track_id = d["id"]
|
||||
if d.get("artist_ids"):
|
||||
self.spotify_artist_ids = list(d["artist_ids"])
|
||||
if d.get("album_image_url"):
|
||||
self.album_image_url = d["album_image_url"]
|
||||
if d.get("release_date"):
|
||||
self.release_date = d["release_date"]
|
||||
if d.get("release_date_precision"):
|
||||
self.release_date_precision = d["release_date_precision"]
|
||||
if d.get("genres"):
|
||||
self.genres = list(d["genres"])
|
||||
if d.get("duration_ms"):
|
||||
self.duration_ms = int(d["duration_ms"])
|
||||
if d.get("track_number"):
|
||||
self.track_number = int(d["track_number"])
|
||||
if d.get("disc_number"):
|
||||
self.disc_number = int(d["disc_number"])
|
||||
if not self.source:
|
||||
self.source = "spotify"
|
||||
return self
|
||||
|
||||
def merge_analysis(self, analysis: dict) -> "Track":
|
||||
"""Merge fields from `maple.analyze.analyze_audio`'s output."""
|
||||
if analysis.get("bpm") is not None:
|
||||
self.bpm = float(analysis["bpm"])
|
||||
return self
|
||||
|
||||
# ---- export adapter -------------------------------------------------- #
|
||||
|
||||
def to_pdb_row(
|
||||
self,
|
||||
*,
|
||||
track_id: int,
|
||||
artist_id: int,
|
||||
album_id: int,
|
||||
genre_id: int = 0,
|
||||
artwork_id: int = 0,
|
||||
file_path: str,
|
||||
analyze_path: str = "",
|
||||
date_added: str = "",
|
||||
tempo_x100: int | None = None,
|
||||
) -> model.Track:
|
||||
"""Project this canonical Track into the denormalised pdb row shape.
|
||||
|
||||
FK ids (`artist_id`, `album_id`, …) and the per-export `file_path` /
|
||||
`analyze_path` are caller-supplied because they only exist in the
|
||||
context of a specific export. `tempo_x100` defaults to `round(bpm *
|
||||
100)` when we have a BPM; pass through otherwise.
|
||||
"""
|
||||
if tempo_x100 is None:
|
||||
tempo_x100 = round(self.bpm * 100) if self.bpm else 0
|
||||
return model.Track(
|
||||
id=track_id,
|
||||
title=self.title,
|
||||
artist_id=artist_id,
|
||||
album_id=album_id,
|
||||
filename=Path(file_path).name,
|
||||
file_path=file_path,
|
||||
file_size=self.file_size,
|
||||
sample_rate=self.sample_rate or 44100,
|
||||
sample_depth=self.sample_depth or 16,
|
||||
bitrate=self.bitrate,
|
||||
duration=self.duration_s,
|
||||
track_number=self.track_number,
|
||||
year=self.year,
|
||||
disc_number=self.disc_number,
|
||||
tempo=tempo_x100,
|
||||
release_date=self.release_date,
|
||||
date_added=date_added,
|
||||
analyze_path=analyze_path,
|
||||
analyze_date=date_added,
|
||||
artwork_id=artwork_id,
|
||||
genre_id=genre_id,
|
||||
source_path=self.path,
|
||||
)
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Cross-cutting per-track validator.
|
||||
|
||||
A track is "valid" iff every export criterion holds. Both `just collection`
|
||||
(`Collection.process_track`) and `just export` (`cli.build`) must invoke
|
||||
this — there is no separate "collection-only" or "export-only" validation
|
||||
in maple. Add new criteria in one place; both pipelines pick them up.
|
||||
|
||||
Criteria (extend as they emerge):
|
||||
- ffprobe-decodable
|
||||
- duration > 0
|
||||
- title / artist / album / date / genre / isrc / bpm tags present
|
||||
- embedded cover art present
|
||||
- analytical BPM cached or fresh (cache key = ISRC)
|
||||
- if `spotify_track` supplied: tags within fuzzy match of Spotify truth
|
||||
|
||||
`validate_track` returns a list of `TrackIssue`. Empty list = passing.
|
||||
Callers decide whether to warn-and-continue (the default) or fail.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from loguru import logger
|
||||
from mutagen import File as MutagenFile
|
||||
|
||||
from maple.analyze import cache_path_for
|
||||
from maple.collection.index import (
|
||||
first_artist,
|
||||
fuzzy_ratio,
|
||||
norm,
|
||||
norm_loose,
|
||||
)
|
||||
from maple.collection.validate import (
|
||||
DURATION_TOLERANCE_S,
|
||||
ffprobe_ok,
|
||||
file_duration,
|
||||
has_embedded_art,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TrackIssue:
|
||||
"""A single validation finding.
|
||||
|
||||
Shape is (playlist, artist, title, reason) — `reason` is a
|
||||
`"<field>: <detail>"` string so structured fields and free-text
|
||||
descriptions share one container.
|
||||
"""
|
||||
|
||||
playlist: str
|
||||
artist: str
|
||||
title: str
|
||||
reason: str
|
||||
|
||||
|
||||
REQUIRED_TAGS = ("title", "artist", "album", "date", "genre", "bpm", "isrc")
|
||||
|
||||
|
||||
def _file_tags(path: Path) -> dict[str, str] | None:
|
||||
try:
|
||||
audio = MutagenFile(path, easy=True)
|
||||
except Exception as e:
|
||||
logger.warning("could not open {} for validation: {}", path.name, e)
|
||||
return None
|
||||
if audio is None or audio.tags is None:
|
||||
return {k: "" for k in REQUIRED_TAGS}
|
||||
tags = audio.tags
|
||||
return {k: (tags.get(k) or [""])[0] if hasattr(tags, "get") else "" for k in REQUIRED_TAGS}
|
||||
|
||||
|
||||
def validate_track(
|
||||
path: Path,
|
||||
*,
|
||||
spotify_track: dict | None = None,
|
||||
analysis_cache_dir: Path | None = None,
|
||||
playlist: str = "",
|
||||
) -> list[TrackIssue]:
|
||||
"""Run every criterion against `path`. Returns the list of failed checks.
|
||||
|
||||
Optional context broadens the checks:
|
||||
- `spotify_track`: enables fuzzy tag comparison against Spotify truth.
|
||||
- `analysis_cache_dir`: enables the "analysis cached" check via ISRC.
|
||||
- `playlist`: tags every issue with playlist context (collection flow).
|
||||
"""
|
||||
tags = _file_tags(path)
|
||||
if tags is None:
|
||||
return [TrackIssue(playlist, "", path.name, "tags: could not read file")]
|
||||
|
||||
artist = tags.get("artist") or (
|
||||
", ".join(spotify_track.get("artists") or []) if spotify_track else ""
|
||||
)
|
||||
title = tags.get("title") or (spotify_track.get("title") if spotify_track else "") or path.stem
|
||||
issues: list[TrackIssue] = []
|
||||
|
||||
def fail(field: str, detail: str) -> None:
|
||||
issues.append(TrackIssue(playlist, artist, title, f"{field}: {detail}"))
|
||||
|
||||
# ---- audio integrity --------------------------------------------------- #
|
||||
if not ffprobe_ok(path):
|
||||
fail("decode", f"ffprobe rejected {path.name}")
|
||||
duration = file_duration(path)
|
||||
if duration <= 0:
|
||||
fail("duration", "≤ 0s — corrupt file or unreadable container")
|
||||
|
||||
# ---- required tags ----------------------------------------------------- #
|
||||
for key in REQUIRED_TAGS:
|
||||
if not (tags.get(key) or "").strip():
|
||||
fail(key, "tag missing")
|
||||
|
||||
# ---- artwork ----------------------------------------------------------- #
|
||||
if not has_embedded_art(path):
|
||||
fail("artwork", "no embedded cover art")
|
||||
|
||||
# ---- analysis cached --------------------------------------------------- #
|
||||
isrc = (tags.get("isrc") or "").strip()
|
||||
if analysis_cache_dir is not None and isrc:
|
||||
cp = cache_path_for(analysis_cache_dir, isrc)
|
||||
if cp is None or not cp.exists():
|
||||
fail("analysis", f"no cached analysis at {cp}")
|
||||
|
||||
# ---- Spotify reconciliation -------------------------------------------- #
|
||||
if spotify_track is not None:
|
||||
sp_artist = ", ".join(spotify_track.get("artists") or [])
|
||||
sp_title = (spotify_track.get("title") or "").strip()
|
||||
sp_album = (spotify_track.get("album") or "").strip()
|
||||
sp_isrc = (spotify_track.get("isrc") or "").strip()
|
||||
sp_duration = (spotify_track.get("duration_ms") or 0) / 1000.0
|
||||
sp_date = (spotify_track.get("release_date") or "").strip()
|
||||
|
||||
f_artist = tags.get("artist") or ""
|
||||
f_title = tags.get("title") or ""
|
||||
f_album = tags.get("album") or ""
|
||||
f_isrc = tags.get("isrc") or ""
|
||||
f_date = tags.get("date") or ""
|
||||
|
||||
if sp_artist and fuzzy_ratio(norm(first_artist(sp_artist)), norm(first_artist(f_artist))) < 0.5:
|
||||
fail("artist", f"spotify={sp_artist!r} file={f_artist!r}")
|
||||
if sp_title and fuzzy_ratio(norm_loose(sp_title), norm_loose(f_title)) < 0.5:
|
||||
fail("title", f"spotify={sp_title!r} file={f_title!r}")
|
||||
if sp_album and f_album and fuzzy_ratio(norm_loose(sp_album), norm_loose(f_album)) < 0.5:
|
||||
fail("album", f"spotify={sp_album!r} file={f_album!r}")
|
||||
elif sp_album and not f_album:
|
||||
fail("album", f"spotify={sp_album!r} file=''")
|
||||
if sp_isrc and f_isrc and sp_isrc.upper() != f_isrc.upper():
|
||||
fail("isrc", f"spotify={sp_isrc!r} file={f_isrc!r}")
|
||||
if sp_date and f_date and sp_date[:4] != f_date[:4]:
|
||||
fail("date", f"spotify={sp_date!r} file={f_date!r}")
|
||||
if sp_duration > 0 and abs(duration - sp_duration) > DURATION_TOLERANCE_S:
|
||||
fail("duration", f"file {duration:.1f}s vs spotify {sp_duration:.1f}s")
|
||||
|
||||
return issues
|
||||
|
||||
|
||||
def log_issues(issues: list[TrackIssue], *, report: bool = True) -> None:
|
||||
"""Emit every issue via loguru.warning. When `report=True`, also bind
|
||||
`extra["report"]` so any attached file-sink (Collection.attach_report_sink
|
||||
or the export-side equivalent) captures the line."""
|
||||
bound = logger.bind(report=True) if report else logger
|
||||
for i in issues:
|
||||
prefix = f"[{i.playlist}] " if i.playlist else ""
|
||||
bound.warning("{}{} - {}: {}", prefix, i.artist, i.title, i.reason)
|
||||
Generated
+41
@@ -90,6 +90,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aubio"
|
||||
version = "0.4.9"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "numpy" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/cd/80/302d89240603e5347c7f8026c8b02c59f8dfaec66c91a743d82de7c86006/aubio-0.4.9.tar.gz", hash = "sha256:df1244f6c4cf5bea382c8c2d35aa43bc31f4cf631fe325ae3992c219546a4202", size = 479008, upload-time = "2019-02-08T11:21:02.438Z" }
|
||||
|
||||
[[package]]
|
||||
name = "bidict"
|
||||
version = "0.23.1"
|
||||
@@ -243,10 +252,12 @@ version = "0.1.0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "aiohttp" },
|
||||
{ name = "aubio" },
|
||||
{ name = "kaitaistruct" },
|
||||
{ name = "loguru" },
|
||||
{ name = "mutagen" },
|
||||
{ name = "pillow" },
|
||||
{ name = "polars" },
|
||||
{ name = "pyrekordbox" },
|
||||
]
|
||||
|
||||
@@ -258,10 +269,12 @@ dev = [
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "aiohttp", specifier = ">=3.10" },
|
||||
{ name = "aubio", specifier = ">=0.4.9" },
|
||||
{ name = "kaitaistruct", specifier = ">=0.11" },
|
||||
{ name = "loguru", specifier = ">=0.7" },
|
||||
{ name = "mutagen", specifier = ">=1.47.0" },
|
||||
{ name = "pillow", specifier = ">=12.2.0" },
|
||||
{ name = "polars", specifier = ">=1.0" },
|
||||
{ name = "pyrekordbox", specifier = ">=0.4.4" },
|
||||
]
|
||||
|
||||
@@ -402,6 +415,34 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "polars"
|
||||
version = "1.41.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "polars-runtime-32" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ff/f9/aeda46259b0669247a160315d2d51269de9504b9dd2f70acadbcb22f46b7/polars-1.41.2.tar.gz", hash = "sha256:256d6731162371b77f3f29a55eacb8c0fc740ddb1a293a01d2ef5b5393c5c708", size = 737996, upload-time = "2026-05-29T17:39:15.604Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1f/22/28f62d24f7db56ac4343588f9362d49b7b4177e55ac47a466fe696b0099b/polars-1.41.2-py3-none-any.whl", hash = "sha256:23ce9a2910b6e3e8d4258770bf44aa17170958df7af6e85feedf4458a04d8d29", size = 833445, upload-time = "2026-05-29T17:37:05.576Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "polars-runtime-32"
|
||||
version = "1.41.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f9/56/54e3ea0e9b64f327179049e4742241cc6b1d3e8fa414b05a057dd26df367/polars_runtime_32-1.41.2.tar.gz", hash = "sha256:7af09ec1ab053da2c9669e8d15f809a4083a29be05db57111688b8051062af56", size = 2989474, upload-time = "2026-05-29T17:39:17.257Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/9b/fe72a3811c0357cdb06c67bdc7695fa1623ad47948fc523195f5ac31037f/polars_runtime_32-1.41.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:95a08346dac337357cdb825c8076df7d36da54c4caa59a5cb41d0a30691c5edd", size = 52265283, upload-time = "2026-05-29T17:37:09.407Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0a/93/fab9da803fd80d9e83ef88c20932f637a10bc611b20415fc322eec84bc44/polars_runtime_32-1.41.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:dedfaeec2c7f995298da7319dd9431d662e5dd1d0ec51b1459df4a0234ceff52", size = 46571222, upload-time = "2026-05-29T17:37:13.698Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/2a/8843f34a8ac57acd058a39b87b03b580dd352a490e9dae0415e02033bdd4/polars_runtime_32-1.41.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18eea22c5cc34e27f8a60950458ad81e6a9ea75e89363ca1367e14e7e7f781fc", size = 50409372, upload-time = "2026-05-29T17:37:17.875Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6c/c6/92b352fe88cf51bd0a19fb99e1c0cbe46aa26c14dcf7995b89869cd932ae/polars_runtime_32-1.41.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2630540dfdfb0f36f9b04a07c7c2e3f50bf2ad384113263c1c812007ee9141e0", size = 56405484, upload-time = "2026-05-29T17:37:22.684Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/c4/bae3174c3b02f6b441d2e58594387abcd509f67a098f682a83b195f08966/polars_runtime_32-1.41.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:20e969e08f9b137e233c04cc04de73d9795f89eb77d34854e40a025965a43763", size = 50603512, upload-time = "2026-05-29T17:37:27.422Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/ed/f2d26ae02d92c2689056838ed59e2a626326ad23c2831d58637d25f6c82a/polars_runtime_32-1.41.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e7016a3deb641b64a31447abbbee0f34bd020a6a9ae34ee6b743837def15e2a4", size = 54328561, upload-time = "2026-05-29T17:37:32.587Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9b/c4/9c3831cc885dc7769e59abf8f583821a5fb4403fd0e4eba0ccc6d47a3d4b/polars_runtime_32-1.41.2-cp310-abi3-win_amd64.whl", hash = "sha256:1e5e5377c315e0dcafdfb2a31adc546abbaeb3f9cb1864e6536523d2af473265", size = 51978643, upload-time = "2026-05-29T17:37:37.443Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/c6/79e9f3f270270d7ed5575d92b7bfef49f01abd9275447161275b23b553a8/polars_runtime_32-1.41.2-cp310-abi3-win_arm64.whl", hash = "sha256:843d96f69d18eca53429c1198e58891db7f18111f83b9c419bb45ad9d73eaed5", size = 46006901, upload-time = "2026-05-29T17:37:42.522Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "propcache"
|
||||
version = "0.5.2"
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
# maple-ts
|
||||
|
||||
TypeScript/Bun port of `maple-py`. Builds a rekordbox `export.pdb` USB layout from a folder of audio files and `.m3u8` playlists.
|
||||
|
||||
## Install
|
||||
|
||||
```sh
|
||||
bun install
|
||||
```
|
||||
|
||||
## Run
|
||||
|
||||
```sh
|
||||
bun run src/cli.ts ./data/input ./data/usb
|
||||
```
|
||||
|
||||
Requires `ffmpeg` on `PATH` for FLAC -> MP3 conversion.
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"name": "maple",
|
||||
"version": "0.1.0",
|
||||
"description": "Build a rekordbox USB from a folder of audio + m3u8 playlists",
|
||||
"type": "module",
|
||||
"module": "src/cli.ts",
|
||||
"bin": {
|
||||
"maple": "src/cli.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"maple": "bun run src/cli.ts",
|
||||
"test": "bun test"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"@types/node-id3": "^0.2.6"
|
||||
},
|
||||
"dependencies": {
|
||||
"music-metadata": "^10.5.0",
|
||||
"node-id3": "^0.2.6"
|
||||
}
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
/** maple CLI: scan a directory of audio + m3u8 playlists and build a rekordbox USB. */
|
||||
import { stat } from "node:fs/promises";
|
||||
import { basename, extname, join, resolve } from "node:path";
|
||||
import os from "node:os";
|
||||
|
||||
import { convertFlacToMp3, verifyDuration } from "./convert.ts";
|
||||
import { stageAudio, stagedPath } from "./copy.ts";
|
||||
import { readTags } from "./meta.ts";
|
||||
import { newDatabase, newTrack, type Database } from "./model.ts";
|
||||
import { writePdb } from "./pdb/writer.ts";
|
||||
import { scanAudio, scanPlaylists } from "./scan.ts";
|
||||
|
||||
function today(): string {
|
||||
return new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function withSuffix(file: string, newExt: string): string {
|
||||
const ext = extname(file);
|
||||
return file.slice(0, file.length - ext.length) + newExt;
|
||||
}
|
||||
|
||||
async function isDir(p: string): Promise<boolean> {
|
||||
try { return (await stat(p)).isDirectory(); } catch { return false; }
|
||||
}
|
||||
|
||||
async function pMap<T, R>(items: T[], workers: number, fn: (item: T) => Promise<R>): Promise<R[]> {
|
||||
const out: R[] = new Array(items.length);
|
||||
let i = 0;
|
||||
const runners = Array.from({ length: Math.max(1, Math.min(workers, items.length)) }, async () => {
|
||||
while (true) {
|
||||
const idx = i++;
|
||||
if (idx >= items.length) return;
|
||||
out[idx] = await fn(items[idx]!);
|
||||
}
|
||||
});
|
||||
await Promise.all(runners);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function build(inputDir: string, usbRoot: string): Promise<Database> {
|
||||
const db = newDatabase();
|
||||
const dateAdded = today();
|
||||
|
||||
const artistIndex = new Map<string, number>();
|
||||
const albumIndex = new Map<string, number>();
|
||||
const trackBySrc = new Map<string, number>();
|
||||
|
||||
const audioFiles = await scanAudio(inputDir);
|
||||
|
||||
const initialMeta = new Map<string, Awaited<ReturnType<typeof readTags>>>();
|
||||
for (const p of audioFiles) initialMeta.set(p, await readTags(p));
|
||||
|
||||
const flacJobs: Array<{ src: string; dst: string }> = [];
|
||||
for (const original of audioFiles) {
|
||||
if (extname(original).toLowerCase() === ".flac") {
|
||||
const m = initialMeta.get(original)!;
|
||||
const mp3Name = withSuffix(basename(original), ".mp3");
|
||||
const { destAbs } = stagedPath(usbRoot, m.artist, m.album, mp3Name);
|
||||
flacJobs.push({ src: original, dst: destAbs });
|
||||
}
|
||||
}
|
||||
|
||||
const workers = Math.max(1, os.cpus().length || 1);
|
||||
const results = await pMap(flacJobs, workers, async (job) => {
|
||||
const converted = await convertFlacToMp3(job.src, job.dst);
|
||||
return { ...job, converted };
|
||||
});
|
||||
for (const r of results) {
|
||||
const rel = r.src.startsWith(inputDir) ? r.src.slice(inputDir.length).replace(/^[\/\\]+/, "") : r.src;
|
||||
console.log(`${r.converted ? "convert" : "cached "} ${rel}`);
|
||||
await verifyDuration(r.src, r.dst);
|
||||
}
|
||||
|
||||
for (const original of audioFiles) {
|
||||
let meta = initialMeta.get(original)!;
|
||||
let artistId = artistIndex.get(meta.artist);
|
||||
if (artistId === undefined) {
|
||||
artistId = db.artists.length + 1;
|
||||
artistIndex.set(meta.artist, artistId);
|
||||
db.artists.push({ id: artistId, name: meta.artist });
|
||||
}
|
||||
const albumKey = `${artistId}\x00${meta.album}`;
|
||||
let albumId = albumIndex.get(albumKey);
|
||||
if (albumId === undefined) {
|
||||
albumId = db.albums.length + 1;
|
||||
albumIndex.set(albumKey, albumId);
|
||||
db.albums.push({ id: albumId, name: meta.album, artist_id: artistId });
|
||||
}
|
||||
|
||||
let filePath: string;
|
||||
if (extname(original).toLowerCase() === ".flac") {
|
||||
const mp3Name = withSuffix(basename(original), ".mp3");
|
||||
const s = stagedPath(usbRoot, meta.artist, meta.album, mp3Name);
|
||||
filePath = s.filePath;
|
||||
meta = await readTags(s.destAbs);
|
||||
} else {
|
||||
const s = await stageAudio(original, usbRoot, meta.artist, meta.album);
|
||||
filePath = s.filePath;
|
||||
}
|
||||
|
||||
const trackId = db.tracks.length + 1;
|
||||
db.tracks.push(newTrack({
|
||||
id: trackId,
|
||||
title: meta.title,
|
||||
artist_id: artistId,
|
||||
album_id: albumId,
|
||||
filename: basename(filePath),
|
||||
file_path: filePath,
|
||||
file_size: meta.file_size,
|
||||
sample_rate: meta.sample_rate,
|
||||
sample_depth: meta.sample_depth,
|
||||
bitrate: meta.bitrate,
|
||||
duration: meta.duration,
|
||||
track_number: meta.track_number,
|
||||
year: meta.year,
|
||||
date_added: dateAdded,
|
||||
source_path: original,
|
||||
}));
|
||||
trackBySrc.set(resolve(original), trackId);
|
||||
}
|
||||
|
||||
const playlists = await scanPlaylists(inputDir);
|
||||
let sortOrder = 1;
|
||||
for (const [name, paths] of playlists) {
|
||||
const playlistId = db.playlist_nodes.length + 1;
|
||||
db.playlist_nodes.push({
|
||||
id: playlistId,
|
||||
parent_id: 0,
|
||||
sort_order: sortOrder,
|
||||
name,
|
||||
is_folder: false,
|
||||
});
|
||||
sortOrder += 1;
|
||||
let entryIndex = 1;
|
||||
for (const p of paths) {
|
||||
const tid = trackBySrc.get(resolve(p));
|
||||
if (tid === undefined) {
|
||||
console.error(`warning: playlist '${name}' references missing file ${p}`);
|
||||
continue;
|
||||
}
|
||||
db.playlist_entries.push({
|
||||
playlist_id: playlistId,
|
||||
track_id: tid,
|
||||
entry_index: entryIndex,
|
||||
});
|
||||
entryIndex += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return db;
|
||||
}
|
||||
|
||||
async function main(argv: string[]): Promise<number> {
|
||||
const args = argv.slice(2);
|
||||
const inputDir = args[0] ?? "./data/input";
|
||||
const outputRoot = args[1] ?? "./data/usb";
|
||||
|
||||
if (!(await isDir(inputDir))) {
|
||||
console.error(`error: input_dir ${inputDir} is not a directory`);
|
||||
return 2;
|
||||
}
|
||||
|
||||
const db = await build(inputDir, outputRoot);
|
||||
const pdbPath = join(outputRoot, "PIONEER", "rekordbox", "export.pdb");
|
||||
await writePdb(db, pdbPath);
|
||||
|
||||
console.log(
|
||||
`wrote ${db.tracks.length} tracks, ${db.artists.length} artists, ${db.albums.length} albums, ` +
|
||||
`${db.playlist_nodes.length} playlists (${db.playlist_entries.length} entries) to ${pdbPath}`
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
process.exit(await main(process.argv));
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
/**
|
||||
* FLAC -> MP3 V0 joint-stereo conversion via ffmpeg, with source-hash dedup.
|
||||
*
|
||||
* The SHA-256 of the source FLAC is stamped into a TXXX:MAPLE_SRC_SHA256 ID3
|
||||
* frame on the produced MP3. Reruns skip conversion when the existing MP3
|
||||
* carries the matching hash, so the cache is content-addressed against inputs.
|
||||
*/
|
||||
import { mkdir, access } from "node:fs/promises";
|
||||
import { dirname } from "node:path";
|
||||
import { parseFile } from "music-metadata";
|
||||
import NodeID3 from "node-id3";
|
||||
|
||||
const HASH_FRAME_DESC = "MAPLE_SRC_SHA256";
|
||||
|
||||
export async function fileSha256(path: string): Promise<string> {
|
||||
const hasher = new Bun.CryptoHasher("sha256");
|
||||
const stream = Bun.file(path).stream();
|
||||
for await (const chunk of stream) hasher.update(chunk);
|
||||
return hasher.digest("hex");
|
||||
}
|
||||
|
||||
async function exists(p: string): Promise<boolean> {
|
||||
try { await access(p); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
function readSourceHash(mp3: string): string | null {
|
||||
const tags = NodeID3.read(mp3);
|
||||
const frames = tags.userDefinedText;
|
||||
if (!frames) return null;
|
||||
for (const f of frames) {
|
||||
if (f.description === HASH_FRAME_DESC && f.value) return f.value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function writeSourceHash(mp3: string, sha: string): void {
|
||||
NodeID3.update({ userDefinedText: [{ description: HASH_FRAME_DESC, value: sha }] }, mp3);
|
||||
}
|
||||
|
||||
async function which(cmd: string): Promise<boolean> {
|
||||
const p = Bun.spawn({ cmd: ["which", cmd], stdout: "pipe", stderr: "pipe" });
|
||||
await p.exited;
|
||||
return p.exitCode === 0;
|
||||
}
|
||||
|
||||
export async function convertFlacToMp3(flac: string, mp3: string): Promise<boolean> {
|
||||
const srcHash = await fileSha256(flac);
|
||||
if (await exists(mp3)) {
|
||||
try {
|
||||
if (readSourceHash(mp3) === srcHash) return false;
|
||||
} catch {}
|
||||
}
|
||||
|
||||
if (!(await which("ffmpeg"))) throw new Error("ffmpeg not found on PATH");
|
||||
|
||||
await mkdir(dirname(mp3), { recursive: true });
|
||||
const proc = Bun.spawn({
|
||||
cmd: [
|
||||
"ffmpeg", "-nostdin", "-loglevel", "error", "-y",
|
||||
"-i", flac,
|
||||
"-codec:a", "libmp3lame",
|
||||
"-q:a", "0",
|
||||
"-joint_stereo", "1",
|
||||
"-map_metadata", "0",
|
||||
mp3,
|
||||
],
|
||||
stdout: "inherit",
|
||||
stderr: "inherit",
|
||||
});
|
||||
await proc.exited;
|
||||
if (proc.exitCode !== 0) throw new Error(`ffmpeg failed for ${flac}`);
|
||||
writeSourceHash(mp3, srcHash);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function duration(path: string): Promise<number> {
|
||||
try {
|
||||
const md = await parseFile(path, { duration: true });
|
||||
return md.format.duration ?? 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyDuration(src: string, dst: string, tolerance = 0.1): Promise<void> {
|
||||
const [a, b] = await Promise.all([duration(src), duration(dst)]);
|
||||
if (Math.abs(a - b) > tolerance) {
|
||||
throw new Error(`duration mismatch: ${src} = ${a.toFixed(3)}s vs ${dst} = ${b.toFixed(3)}s (Δ ${Math.abs(a - b).toFixed(3)}s)`);
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
import { dirname, join, basename } from "node:path";
|
||||
import { mkdir, copyFile, access } from "node:fs/promises";
|
||||
|
||||
const BAD = /[\\/:*?"<>|\x00-\x1f]/g;
|
||||
|
||||
export function sanitise(component: string): string {
|
||||
const s = component.replace(BAD, "_").replace(/^[ .]+|[ .]+$/g, "");
|
||||
return s || "_";
|
||||
}
|
||||
|
||||
export function stagedPath(usbRoot: string, artist: string, album: string, filename: string): { destAbs: string; filePath: string } {
|
||||
const relParts = ["Contents", sanitise(artist), sanitise(album), sanitise(filename)];
|
||||
const destAbs = join(usbRoot, ...relParts);
|
||||
const filePath = "/" + relParts.join("/");
|
||||
return { destAbs, filePath };
|
||||
}
|
||||
|
||||
async function exists(p: string): Promise<boolean> {
|
||||
try { await access(p); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
export async function stageAudio(src: string, usbRoot: string, artist: string, album: string): Promise<{ destAbs: string; filePath: string }> {
|
||||
const { destAbs, filePath } = stagedPath(usbRoot, artist, album, basename(src));
|
||||
await mkdir(dirname(destAbs), { recursive: true });
|
||||
if (!(await exists(destAbs))) {
|
||||
await copyFile(src, destAbs);
|
||||
}
|
||||
return { destAbs, filePath };
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
import { parseFile } from "music-metadata";
|
||||
import { basename, extname } from "node:path";
|
||||
import { stat } from "node:fs/promises";
|
||||
|
||||
export interface TrackMeta {
|
||||
title: string;
|
||||
artist: string;
|
||||
album: string;
|
||||
track_number: number;
|
||||
year: number;
|
||||
duration: number;
|
||||
bitrate: number;
|
||||
sample_rate: number;
|
||||
sample_depth: number;
|
||||
file_size: number;
|
||||
}
|
||||
|
||||
function toInt(v: unknown): number {
|
||||
if (v === null || v === undefined) return 0;
|
||||
let s = String(Array.isArray(v) ? v[0] : v).trim();
|
||||
if (!s) return 0;
|
||||
s = s.split("/")[0]!;
|
||||
const n = parseInt(s, 10);
|
||||
return Number.isFinite(n) ? n : 0;
|
||||
}
|
||||
|
||||
export async function readTags(path: string): Promise<TrackMeta> {
|
||||
const { size: file_size } = await stat(path);
|
||||
const stem = basename(path, extname(path));
|
||||
try {
|
||||
const md = await parseFile(path, { duration: true });
|
||||
const c = md.common;
|
||||
const info = md.format;
|
||||
const bitrate_bps = info.bitrate ?? 0;
|
||||
const bitrate = bitrate_bps > 1000 ? Math.floor(bitrate_bps / 1000) : bitrate_bps;
|
||||
return {
|
||||
title: c.title || stem,
|
||||
artist: c.artist || "Unknown Artist",
|
||||
album: c.album || "Unknown Album",
|
||||
track_number: toInt(c.track?.no),
|
||||
year: toInt(c.year ?? c.date),
|
||||
duration: Math.floor(info.duration ?? 0),
|
||||
bitrate,
|
||||
sample_rate: info.sampleRate ?? 44100,
|
||||
sample_depth: info.bitsPerSample ?? 16,
|
||||
file_size,
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
title: stem,
|
||||
artist: "Unknown Artist",
|
||||
album: "Unknown Album",
|
||||
track_number: 0,
|
||||
year: 0,
|
||||
duration: 0,
|
||||
bitrate: 0,
|
||||
sample_rate: 44100,
|
||||
sample_depth: 16,
|
||||
file_size,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
export interface Artist {
|
||||
id: number;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface Album {
|
||||
id: number;
|
||||
name: string;
|
||||
artist_id: number;
|
||||
}
|
||||
|
||||
export interface Track {
|
||||
id: number;
|
||||
title: string;
|
||||
artist_id: number;
|
||||
album_id: number;
|
||||
filename: string;
|
||||
file_path: string;
|
||||
file_size: number;
|
||||
sample_rate: number;
|
||||
sample_depth: number;
|
||||
bitrate: number;
|
||||
duration: number;
|
||||
track_number: number;
|
||||
year: number;
|
||||
disc_number: number;
|
||||
tempo: number;
|
||||
comment: string;
|
||||
date_added: string;
|
||||
release_date: string;
|
||||
mix_name: string;
|
||||
analyze_path: string;
|
||||
analyze_date: string;
|
||||
source_path: string | null;
|
||||
}
|
||||
|
||||
export interface PlaylistNode {
|
||||
id: number;
|
||||
parent_id: number;
|
||||
sort_order: number;
|
||||
name: string;
|
||||
is_folder: boolean;
|
||||
}
|
||||
|
||||
export interface PlaylistEntry {
|
||||
playlist_id: number;
|
||||
track_id: number;
|
||||
entry_index: number;
|
||||
}
|
||||
|
||||
export interface Database {
|
||||
tracks: Track[];
|
||||
artists: Artist[];
|
||||
albums: Album[];
|
||||
playlist_nodes: PlaylistNode[];
|
||||
playlist_entries: PlaylistEntry[];
|
||||
}
|
||||
|
||||
export function newDatabase(): Database {
|
||||
return { tracks: [], artists: [], albums: [], playlist_nodes: [], playlist_entries: [] };
|
||||
}
|
||||
|
||||
export function newTrack(init: Pick<Track, "id" | "title" | "artist_id" | "album_id" | "filename" | "file_path"> & Partial<Track>): Track {
|
||||
return {
|
||||
file_size: 0,
|
||||
sample_rate: 44100,
|
||||
sample_depth: 16,
|
||||
bitrate: 0,
|
||||
duration: 0,
|
||||
track_number: 0,
|
||||
year: 0,
|
||||
disc_number: 0,
|
||||
tempo: 0,
|
||||
comment: "",
|
||||
date_added: "",
|
||||
release_date: "",
|
||||
mix_name: "",
|
||||
analyze_path: "",
|
||||
analyze_date: "",
|
||||
source_path: null,
|
||||
...init,
|
||||
};
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
/** Small little-endian write helpers backed by Uint8Array. */
|
||||
export class LE {
|
||||
parts: Uint8Array[] = [];
|
||||
private len = 0;
|
||||
|
||||
push(u: Uint8Array): void { this.parts.push(u); this.len += u.length; }
|
||||
|
||||
u8(v: number): void {
|
||||
const b = new Uint8Array(1); b[0] = v & 0xff; this.push(b);
|
||||
}
|
||||
u16(v: number): void {
|
||||
const b = new Uint8Array(2);
|
||||
b[0] = v & 0xff;
|
||||
b[1] = (v >> 8) & 0xff;
|
||||
this.push(b);
|
||||
}
|
||||
u32(v: number): void {
|
||||
const b = new Uint8Array(4);
|
||||
b[0] = v & 0xff;
|
||||
b[1] = (v >>> 8) & 0xff;
|
||||
b[2] = (v >>> 16) & 0xff;
|
||||
b[3] = (v >>> 24) & 0xff;
|
||||
this.push(b);
|
||||
}
|
||||
bytes(b: Uint8Array): void { this.push(b); }
|
||||
|
||||
length(): number { return this.len; }
|
||||
|
||||
build(): Uint8Array {
|
||||
const out = new Uint8Array(this.len);
|
||||
let o = 0;
|
||||
for (const p of this.parts) { out.set(p, o); o += p.length; }
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
export function concat(parts: Uint8Array[]): Uint8Array {
|
||||
let len = 0;
|
||||
for (const p of parts) len += p.length;
|
||||
const out = new Uint8Array(len);
|
||||
let o = 0;
|
||||
for (const p of parts) { out.set(p, o); o += p.length; }
|
||||
return out;
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
import { LE } from "./buf.ts";
|
||||
|
||||
export const PAGE_SIZE = 4096;
|
||||
const HEADER_SIZE = 40;
|
||||
const ROW_GROUP_SIZE = 36;
|
||||
const ROWS_PER_GROUP = 16;
|
||||
|
||||
function align4(o: number): number { return (o + 3) & ~3; }
|
||||
function numRowGroups(n: number): number { return Math.floor((n + ROWS_PER_GROUP - 1) / ROWS_PER_GROUP); }
|
||||
|
||||
export class PageBuilder {
|
||||
pageIndex: number;
|
||||
pageType: number;
|
||||
nextPage: number;
|
||||
rowBytes: Uint8Array[] = [];
|
||||
rowOffsets: number[] = [];
|
||||
heapCursor = 0;
|
||||
|
||||
constructor(pageIndex: number, pageType: number) {
|
||||
this.pageIndex = pageIndex;
|
||||
this.pageType = pageType;
|
||||
this.nextPage = pageIndex;
|
||||
}
|
||||
|
||||
tryAdd(row: Uint8Array): boolean {
|
||||
const aligned = align4(this.heapCursor);
|
||||
const end = aligned + row.length;
|
||||
const footerAfter = numRowGroups(this.rowBytes.length + 1) * ROW_GROUP_SIZE;
|
||||
if (end + HEADER_SIZE + footerAfter > PAGE_SIZE) return false;
|
||||
this.rowOffsets.push(aligned);
|
||||
this.rowBytes.push(row);
|
||||
this.heapCursor = end;
|
||||
return true;
|
||||
}
|
||||
|
||||
numRows(): number { return this.rowBytes.length; }
|
||||
|
||||
finalize(): Uint8Array {
|
||||
const n = this.rowBytes.length;
|
||||
const numRgs = numRowGroups(n);
|
||||
const footerSize = numRgs * ROW_GROUP_SIZE;
|
||||
const heapPadding = PAGE_SIZE - HEADER_SIZE - footerSize;
|
||||
const usedSize = this.heapCursor;
|
||||
const freeSize = heapPadding - usedSize;
|
||||
if (freeSize < 0) throw new Error("page overflow");
|
||||
|
||||
const pageFlags = n > 0 ? 0x00 : 0x44;
|
||||
|
||||
const h = new LE();
|
||||
h.u32(0); // magic
|
||||
h.u32(this.pageIndex);
|
||||
h.u32(this.pageType);
|
||||
h.u32(this.nextPage);
|
||||
h.u32(0); // unknown1
|
||||
h.u32(0); // unknown2
|
||||
h.u8(n & 0xff); h.u8(0); h.u8(0); h.u8(pageFlags);
|
||||
h.u16(freeSize); h.u16(usedSize);
|
||||
h.u16(0); // unknown5
|
||||
h.u16(n); // num_rows_large
|
||||
h.u16(0); // unknown6
|
||||
h.u16(0); // unknown7
|
||||
const header = h.build();
|
||||
if (header.length !== HEADER_SIZE) throw new Error(`hdr len ${header.length}`);
|
||||
|
||||
const heap = new Uint8Array(heapPadding);
|
||||
for (let i = 0; i < this.rowBytes.length; i++) {
|
||||
heap.set(this.rowBytes[i]!, this.rowOffsets[i]!);
|
||||
}
|
||||
|
||||
const footer = new LE();
|
||||
for (let g = numRgs - 1; g >= 0; g--) {
|
||||
const start = g * ROWS_PER_GROUP;
|
||||
const grpOffsets = this.rowOffsets.slice(start, start + ROWS_PER_GROUP);
|
||||
const grpCount = grpOffsets.length;
|
||||
const padded = grpOffsets.slice();
|
||||
while (padded.length < ROWS_PER_GROUP) padded.push(0);
|
||||
const presence = (1 << grpCount) - 1;
|
||||
const rg = new LE();
|
||||
for (let i = padded.length - 1; i >= 0; i--) rg.u16(padded[i]!);
|
||||
rg.u16(presence);
|
||||
rg.u16(16);
|
||||
const rgBytes = rg.build();
|
||||
if (rgBytes.length !== ROW_GROUP_SIZE) throw new Error("rg size");
|
||||
footer.bytes(rgBytes);
|
||||
}
|
||||
const footerBytes = footer.build();
|
||||
if (footerBytes.length !== footerSize) throw new Error("footer size");
|
||||
|
||||
const out = new Uint8Array(PAGE_SIZE);
|
||||
out.set(header, 0);
|
||||
out.set(heap, HEADER_SIZE);
|
||||
out.set(footerBytes, HEADER_SIZE + heapPadding);
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
import type { Album, Artist, PlaylistEntry, PlaylistNode, Track } from "../model.ts";
|
||||
import { encodeDevicesql } from "./strings.ts";
|
||||
import { LE, concat } from "./buf.ts";
|
||||
|
||||
const TRACK_FIXED_PREFIX = 0x5e;
|
||||
const TRACK_OFS_STRINGS_START = TRACK_FIXED_PREFIX;
|
||||
const TRACK_HEADER_LEN = TRACK_OFS_STRINGS_START + 21 * 2;
|
||||
|
||||
function pad2(b: Uint8Array): Uint8Array {
|
||||
if (b.length & 1) {
|
||||
const out = new Uint8Array(b.length + 1);
|
||||
out.set(b);
|
||||
return out;
|
||||
}
|
||||
return b;
|
||||
}
|
||||
|
||||
export function serializeTrack(t: Track): Uint8Array {
|
||||
const strings: string[] = [
|
||||
"", // 0 isrc
|
||||
"", // 1 texter
|
||||
"1", // 2 unknown_string_2
|
||||
"2", // 3 unknown_string_3
|
||||
"", // 4 unknown_string_4
|
||||
"", // 5 message
|
||||
"ON", // 6 kuvo_public
|
||||
"ON", // 7 autoload_hot_cues
|
||||
"", // 8 unknown_string_5
|
||||
"", // 9 unknown_string_6
|
||||
t.date_added, // 10
|
||||
t.release_date, // 11
|
||||
t.mix_name, // 12
|
||||
"", // 13
|
||||
t.analyze_path, // 14
|
||||
t.analyze_date, // 15
|
||||
t.comment, // 16
|
||||
t.title, // 17
|
||||
"", // 18
|
||||
t.filename, // 19
|
||||
t.file_path, // 20
|
||||
];
|
||||
|
||||
const encoded = strings.map(encodeDevicesql);
|
||||
const offsets: number[] = [];
|
||||
let cursor = TRACK_HEADER_LEN;
|
||||
for (const e of encoded) {
|
||||
offsets.push(cursor);
|
||||
cursor += e.length;
|
||||
}
|
||||
const blob = concat(encoded);
|
||||
|
||||
const p = new LE();
|
||||
p.u16(0x24); p.u16(0); // subtype, index_shift
|
||||
p.u32(788224); // bitmask
|
||||
p.u32(t.sample_rate);
|
||||
p.u32(0); // composer_id
|
||||
p.u32(t.file_size);
|
||||
p.u32(0); // unknown
|
||||
p.u16(19048);
|
||||
p.u16(30967);
|
||||
p.u32(0); // artwork_id
|
||||
p.u32(0); // key_id
|
||||
p.u32(0); // original_artist_id
|
||||
p.u32(0); // label_id
|
||||
p.u32(0); // remixer_id
|
||||
p.u32(t.bitrate);
|
||||
p.u32(t.track_number);
|
||||
p.u32(t.tempo);
|
||||
p.u32(0); // genre_id
|
||||
p.u32(t.album_id);
|
||||
p.u32(t.artist_id);
|
||||
p.u32(t.id);
|
||||
p.u16(t.disc_number);
|
||||
p.u16(0); // play_count
|
||||
p.u16(t.year);
|
||||
p.u16(t.sample_depth);
|
||||
p.u16(t.duration);
|
||||
p.u16(41);
|
||||
p.u8(0); p.u8(0); // color_id, rating
|
||||
p.u16(1);
|
||||
p.u16(3);
|
||||
|
||||
const prefix = p.build();
|
||||
if (prefix.length !== TRACK_FIXED_PREFIX) throw new Error(`prefix len ${prefix.length}`);
|
||||
|
||||
const ofs = new LE();
|
||||
for (const o of offsets) ofs.u16(o);
|
||||
return pad2(concat([prefix, ofs.build(), blob]));
|
||||
}
|
||||
|
||||
export function serializeArtist(a: Artist): Uint8Array {
|
||||
const name = encodeDevicesql(a.name);
|
||||
const nearOfs = 0x0a;
|
||||
const farThreshold = 0x100;
|
||||
const r = new LE();
|
||||
if (nearOfs < farThreshold) {
|
||||
r.u16(0x60); r.u16(a.id & 0xffff);
|
||||
r.u32(a.id);
|
||||
r.u8(0x03);
|
||||
r.u8(nearOfs);
|
||||
if (r.length() !== nearOfs) throw new Error("artist near offset mismatch");
|
||||
r.bytes(name);
|
||||
} else {
|
||||
const far = 0x0e;
|
||||
r.u16(0x64); r.u16(a.id & 0xffff);
|
||||
r.u32(a.id);
|
||||
r.u8(0x03);
|
||||
r.u8(0);
|
||||
r.u16(far);
|
||||
r.bytes(name);
|
||||
}
|
||||
return pad2(r.build());
|
||||
}
|
||||
|
||||
export function serializeAlbum(a: Album): Uint8Array {
|
||||
const name = encodeDevicesql(a.name);
|
||||
const nearOfs = 0x16;
|
||||
const r = new LE();
|
||||
r.u16(0x80); r.u16(a.id & 0xffff);
|
||||
r.u32(0);
|
||||
r.u32(a.artist_id);
|
||||
r.u32(a.id);
|
||||
r.u32(0);
|
||||
r.u8(0x03);
|
||||
r.u8(nearOfs);
|
||||
if (r.length() !== nearOfs) throw new Error("album near offset mismatch");
|
||||
r.bytes(name);
|
||||
return pad2(r.build());
|
||||
}
|
||||
|
||||
export function serializePlaylistTree(n: PlaylistNode): Uint8Array {
|
||||
const r = new LE();
|
||||
r.u32(n.parent_id);
|
||||
r.u32(0);
|
||||
r.u32(n.sort_order);
|
||||
r.u32(n.id);
|
||||
r.u32(n.is_folder ? 1 : 0);
|
||||
r.bytes(encodeDevicesql(n.name));
|
||||
return pad2(r.build());
|
||||
}
|
||||
|
||||
export function serializePlaylistEntry(e: PlaylistEntry): Uint8Array {
|
||||
const r = new LE();
|
||||
r.u32(e.entry_index);
|
||||
r.u32(e.track_id);
|
||||
r.u32(e.playlist_id);
|
||||
return r.build();
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
const EMPTY = new Uint8Array([0x03]);
|
||||
|
||||
function isAscii(s: string): boolean {
|
||||
for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) > 0x7f) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
export function encodeDevicesql(s: string): Uint8Array {
|
||||
if (!s) return EMPTY;
|
||||
if (isAscii(s) && s.length < 127) {
|
||||
const ascii = new TextEncoder().encode(s);
|
||||
const length = ascii.length + 1;
|
||||
const lengthAndKind = ((length << 1) | 1) & 0xff;
|
||||
const out = new Uint8Array(1 + ascii.length);
|
||||
out[0] = lengthAndKind;
|
||||
out.set(ascii, 1);
|
||||
return out;
|
||||
}
|
||||
const utf16 = new Uint8Array(s.length * 2);
|
||||
for (let i = 0; i < s.length; i++) {
|
||||
const c = s.charCodeAt(i);
|
||||
utf16[i * 2] = c & 0xff;
|
||||
utf16[i * 2 + 1] = (c >> 8) & 0xff;
|
||||
}
|
||||
const totalLength = utf16.length + 4;
|
||||
const out = new Uint8Array(4 + utf16.length);
|
||||
out[0] = 0x90;
|
||||
out[1] = totalLength & 0xff;
|
||||
out[2] = (totalLength >> 8) & 0xff;
|
||||
out[3] = 0x00;
|
||||
out.set(utf16, 4);
|
||||
return out;
|
||||
}
|
||||
@@ -1,122 +0,0 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname } from "node:path";
|
||||
import type { Database } from "../model.ts";
|
||||
import { PAGE_SIZE, PageBuilder } from "./pager.ts";
|
||||
import {
|
||||
serializeAlbum,
|
||||
serializeArtist,
|
||||
serializePlaylistEntry,
|
||||
serializePlaylistTree,
|
||||
serializeTrack,
|
||||
} from "./rows.ts";
|
||||
import { LE } from "./buf.ts";
|
||||
|
||||
const PAGE_TYPE_TRACKS = 0;
|
||||
const PAGE_TYPE_ARTISTS = 2;
|
||||
const PAGE_TYPE_ALBUMS = 3;
|
||||
const PAGE_TYPE_PLAYLIST_TREE = 7;
|
||||
const PAGE_TYPE_PLAYLIST_ENTRIES = 8;
|
||||
|
||||
const FILE_HEADER_SIZE = 28;
|
||||
const TABLE_ENTRY_SIZE = 16;
|
||||
|
||||
interface Chain {
|
||||
pageType: number;
|
||||
pages: Uint8Array[];
|
||||
first: number;
|
||||
last: number;
|
||||
}
|
||||
|
||||
function buildChain(pageType: number, rows: Uint8Array[], startIndex: number): Chain {
|
||||
const builders: PageBuilder[] = [];
|
||||
let idx = startIndex;
|
||||
let current = new PageBuilder(idx, pageType);
|
||||
|
||||
for (const row of rows) {
|
||||
if (!current.tryAdd(row)) {
|
||||
builders.push(current);
|
||||
idx += 1;
|
||||
current = new PageBuilder(idx, pageType);
|
||||
if (!current.tryAdd(row)) {
|
||||
throw new Error(`row of size ${row.length} too large for an empty page`);
|
||||
}
|
||||
}
|
||||
}
|
||||
builders.push(current);
|
||||
|
||||
const out: Uint8Array[] = [];
|
||||
for (let i = 0; i < builders.length; i++) {
|
||||
const p = builders[i]!;
|
||||
if (i === builders.length - 1) p.nextPage = p.pageIndex;
|
||||
else p.nextPage = p.pageIndex + 1;
|
||||
out.push(p.finalize());
|
||||
}
|
||||
|
||||
return { pageType, pages: out, first: startIndex, last: idx };
|
||||
}
|
||||
|
||||
export async function writePdb(db: Database, path: string): Promise<void> {
|
||||
const chains: Chain[] = [];
|
||||
let nextIdx = 1;
|
||||
|
||||
const tables: Array<[number, Uint8Array[]]> = [
|
||||
[PAGE_TYPE_TRACKS, db.tracks.map(serializeTrack)],
|
||||
[PAGE_TYPE_ARTISTS, db.artists.map(serializeArtist)],
|
||||
[PAGE_TYPE_ALBUMS, db.albums.map(serializeAlbum)],
|
||||
[PAGE_TYPE_PLAYLIST_TREE, db.playlist_nodes.map(serializePlaylistTree)],
|
||||
[PAGE_TYPE_PLAYLIST_ENTRIES, db.playlist_entries.map(serializePlaylistEntry)],
|
||||
];
|
||||
|
||||
for (const [ptype, rows] of tables) {
|
||||
const c = buildChain(ptype, rows, nextIdx);
|
||||
chains.push(c);
|
||||
nextIdx = c.last + 1;
|
||||
}
|
||||
|
||||
const numTables = chains.length;
|
||||
const nextUnused = nextIdx;
|
||||
|
||||
const h = new LE();
|
||||
h.u32(0);
|
||||
h.u32(PAGE_SIZE);
|
||||
h.u32(numTables);
|
||||
h.u32(nextUnused);
|
||||
h.u32(0);
|
||||
h.u32(1);
|
||||
h.u32(0);
|
||||
const fileHeader = h.build();
|
||||
if (fileHeader.length !== FILE_HEADER_SIZE) throw new Error("file header size");
|
||||
|
||||
const tbl = new LE();
|
||||
for (const c of chains) {
|
||||
tbl.u32(c.pageType);
|
||||
tbl.u32(0);
|
||||
tbl.u32(c.first);
|
||||
tbl.u32(c.last);
|
||||
}
|
||||
const tablesBlock = tbl.build();
|
||||
if (tablesBlock.length !== numTables * TABLE_ENTRY_SIZE) throw new Error("tables size");
|
||||
|
||||
const headerFull = new Uint8Array(fileHeader.length + tablesBlock.length);
|
||||
headerFull.set(fileHeader, 0);
|
||||
headerFull.set(tablesBlock, fileHeader.length);
|
||||
if (headerFull.length > PAGE_SIZE) throw new Error("too many tables for one page");
|
||||
|
||||
const page0 = new Uint8Array(PAGE_SIZE);
|
||||
page0.set(headerFull, 0);
|
||||
|
||||
let totalLen = PAGE_SIZE;
|
||||
for (const c of chains) totalLen += c.pages.length * PAGE_SIZE;
|
||||
const out = new Uint8Array(totalLen);
|
||||
let off = 0;
|
||||
out.set(page0, off); off += PAGE_SIZE;
|
||||
for (const c of chains) {
|
||||
for (const p of c.pages) {
|
||||
if (p.length !== PAGE_SIZE) throw new Error("page size mismatch");
|
||||
out.set(p, off); off += PAGE_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
await writeFile(path, out);
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
import { readdir } from "node:fs/promises";
|
||||
import { join, extname, basename, dirname, isAbsolute, resolve } from "node:path";
|
||||
|
||||
const AUDIO_EXTS = new Set([".mp3", ".flac", ".wav", ".aiff", ".aif", ".m4a"]);
|
||||
|
||||
async function walk(dir: string): Promise<string[]> {
|
||||
const out: string[] = [];
|
||||
const entries = await readdir(dir, { withFileTypes: true });
|
||||
for (const e of entries) {
|
||||
const p = join(dir, e.name);
|
||||
if (e.isDirectory()) {
|
||||
out.push(...(await walk(p)));
|
||||
} else if (e.isFile()) {
|
||||
out.push(p);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function scanAudio(inputDir: string): Promise<string[]> {
|
||||
const all = await walk(inputDir);
|
||||
return all.filter((p) => AUDIO_EXTS.has(extname(p).toLowerCase())).sort();
|
||||
}
|
||||
|
||||
export async function parseM3u8(playlistPath: string): Promise<string[]> {
|
||||
const base = dirname(playlistPath);
|
||||
const text = await Bun.file(playlistPath).text();
|
||||
const entries: string[] = [];
|
||||
for (const raw of text.split(/\r?\n/)) {
|
||||
const s = raw.trim();
|
||||
if (!s || s.startsWith("#")) continue;
|
||||
entries.push(isAbsolute(s) ? resolve(s) : resolve(join(base, s)));
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
export async function scanPlaylists(inputDir: string): Promise<Map<string, string[]>> {
|
||||
const all = await walk(inputDir);
|
||||
const playlists = all.filter((p) => extname(p).toLowerCase() === ".m3u8").sort();
|
||||
const out = new Map<string, string[]>();
|
||||
for (const p of playlists) {
|
||||
const stem = basename(p, extname(p));
|
||||
out.set(stem, await parseM3u8(p));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { encodeDevicesql } from "../src/pdb/strings.ts";
|
||||
|
||||
test("empty encodes to 0x03", () => {
|
||||
expect(Array.from(encodeDevicesql(""))).toEqual([0x03]);
|
||||
});
|
||||
|
||||
test("short ascii uses short form", () => {
|
||||
const out = encodeDevicesql("ABC");
|
||||
// length = 3+1 = 4; (4<<1)|1 = 9
|
||||
expect(out[0]).toBe(9);
|
||||
expect(Array.from(out.slice(1))).toEqual([0x41, 0x42, 0x43]);
|
||||
});
|
||||
|
||||
test("non-ascii uses utf-16 form", () => {
|
||||
const out = encodeDevicesql("é");
|
||||
expect(out[0]).toBe(0x90);
|
||||
// total_length = utf16 bytes (2) + 4 = 6
|
||||
expect(out[1]).toBe(6);
|
||||
expect(out[2]).toBe(0);
|
||||
expect(out[3]).toBe(0);
|
||||
expect(out[4]).toBe(0xe9);
|
||||
expect(out[5]).toBe(0x00);
|
||||
});
|
||||
@@ -1,30 +0,0 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { newDatabase, newTrack } from "../src/model.ts";
|
||||
import { writePdb } from "../src/pdb/writer.ts";
|
||||
import { PAGE_SIZE } from "../src/pdb/pager.ts";
|
||||
|
||||
test("writes a multi-page-aligned pdb file", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "maple-ts-"));
|
||||
try {
|
||||
const db = newDatabase();
|
||||
db.artists.push({ id: 1, name: "A" });
|
||||
db.albums.push({ id: 1, name: "Alb", artist_id: 1 });
|
||||
db.tracks.push(newTrack({
|
||||
id: 1, title: "T", artist_id: 1, album_id: 1,
|
||||
filename: "t.mp3", file_path: "/Contents/A/Alb/t.mp3",
|
||||
}));
|
||||
db.playlist_nodes.push({ id: 1, parent_id: 0, sort_order: 1, name: "P", is_folder: false });
|
||||
db.playlist_entries.push({ playlist_id: 1, track_id: 1, entry_index: 1 });
|
||||
|
||||
const out = join(dir, "export.pdb");
|
||||
await writePdb(db, out);
|
||||
const buf = await readFile(out);
|
||||
expect(buf.length % PAGE_SIZE).toBe(0);
|
||||
expect(buf.length).toBeGreaterThanOrEqual(PAGE_SIZE * 6);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"types": ["bun-types"],
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true,
|
||||
"verbatimModuleSyntax": false
|
||||
},
|
||||
"include": ["src/**/*.ts", "tests/**/*.ts"]
|
||||
}
|
||||
+28
-10
@@ -4,23 +4,41 @@
|
||||
{
|
||||
"path": ".",
|
||||
},
|
||||
{
|
||||
"path": "h:ome\\ntr\\code\\rekordcrate",
|
||||
},
|
||||
{
|
||||
"path": "U:sers\\nathanrashleigh\\code\\code\\rekordcrate",
|
||||
},
|
||||
{
|
||||
"path": "C:\\Users\\ntr\\code\\rekordcrate"
|
||||
},
|
||||
],
|
||||
"settings":
|
||||
{
|
||||
"LSP":
|
||||
{
|
||||
"LSP-pyright":
|
||||
{
|
||||
// Use maple-py/ as the python project root so pyrightconfig.json
|
||||
// there is picked up (extraPaths=["src"], venvPath=".venv",
|
||||
// pythonVersion="3.14"). The interpreter is the uv-managed venv.
|
||||
"settings":
|
||||
{
|
||||
"python.pythonPath": "$folder/maple-py/.venv/bin/python",
|
||||
"python.analysis.extraPaths":
|
||||
[
|
||||
"$folder/maple-py/src",
|
||||
],
|
||||
"python.analysis.typeCheckingMode": "basic",
|
||||
"python.analysis.diagnosticMode": "openFilesOnly",
|
||||
"python.analysis.autoImportCompletions": true,
|
||||
"python.analysis.indexing": true,
|
||||
},
|
||||
// Pin the LSP server's working directory to maple-py/ so file
|
||||
// lookups resolve relative to where pyrightconfig.json lives.
|
||||
"initializationOptions":
|
||||
{
|
||||
"pythonPath": "$folder/maple-py/.venv/bin/python",
|
||||
},
|
||||
},
|
||||
},
|
||||
"cargo_build":
|
||||
{
|
||||
"paths":
|
||||
{
|
||||
"C:\\Users\\ntr\\code\\maple\\maple-rs":
|
||||
"$folder/maple-rs":
|
||||
{
|
||||
"variants":
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user