zzz

maple-py/src/maple/analyze.py

@@ -27,9 +27,17 @@ SAMPLE_RATE = 44100
 MIN_BEATS = 8
 
 # Waveform envelope dimensions.
-PREVIEW_COLS = 400        # PWAV is fixed at 400 columns (1 byte/col).
-SCROLL_FPS = 150          # PWV3 = ~150 columns/sec across the whole track.
-PWAV_LEVELS = 31          # PWAV/PWV3 stores 5-bit heights (bit 5 = white flag).
+# PWAV is stored as 200 col-pairs (height byte + intensity byte) = 400 bytes
+# per Beat Link's WaveformPreview reader. Spec page says 400 cols × 1 byte;
+# Beat Link reads 200 segments × 2 bytes — Beat Link is the production
+# reference. PWV2 follows the same 2-bytes-per-col scheme (50 cols × 2B = 100B)
+# but with 4-bit height (0-15) per the spec.
+PWAV_COLS = 200            # paired-byte columns in PWAV (→ 400 bytes data)
+PWV2_COLS = 50             # paired-byte columns in PWV2 (→ 100 bytes data)
+SCROLL_FPS = 150           # PWV3 / PWV5 / PWV7 = ~150 cols/sec
+PWAV_HEIGHT_LEVELS = 31    # 5-bit (0-31), bits 0-4
+PWV2_HEIGHT_LEVELS = 15    # 4-bit (0-15), bits 0-3
+PWAV_LEVELS = 31           # legacy alias for callers
 
 
 def _stream_pcm(path: Path):
@@ -115,23 +123,83 @@ def _decode_pcm(path: Path) -> np.ndarray:
     return np.frombuffer(proc.stdout, dtype=np.float32)
 
 
-def _bin_amplitudes(samples: np.ndarray, n_bins: int, levels: int = PWAV_LEVELS) -> bytes:
-    """Split `samples` into `n_bins` contiguous windows, take the peak |s| per
-    window, quantise to 0..levels, and return one byte per bin.
+# Real rekordbox waveforms have a sharp noise gate and an aggressive
+# S-curve: silence reads as actual zero, peaks pin to max, mids cluster
+# at the high end. Linear peak-normalisation produces mushy mid-range
+# values that look "always loud, never silent" on the deck. These two
+# numbers were tuned against the data/q1-rekordbox/ reference export.
+SILENCE_GATE_PCT = 0.02     # < 2% of peak → 0  (matches ~13,600 silent cols on Q1)
+QUANT_GAMMA = 0.55          # > sub-linear curve, pushes mid-values toward max
 
-    Quantising at 5-bit resolution matches PWAV's `bits 0-4 = height` encoding;
-    bit 5 (the "white overshoot" flag) is left at 0 for now — most CDJ firmware
-    is happy without it.
-    """
+
+def _quantise(peaks: np.ndarray, levels: int) -> np.ndarray:
+    """Gate + gamma + quantise. Returns uint8 array of length len(peaks)."""
+    peak_max = float(peaks.max()) if peaks.size else 0.0
+    if peak_max <= 0:
+        return np.zeros(peaks.size, dtype=np.uint8)
+    gated = np.where(peaks < peak_max * SILENCE_GATE_PCT, 0.0, peaks)
+    normalised = gated / peak_max
+    curved = np.power(normalised, QUANT_GAMMA, where=normalised > 0)
+    return np.clip(np.round(curved * levels), 0, levels).astype(np.uint8)
+
+
+def _bin_amplitudes(samples: np.ndarray, n_bins: int, levels: int = PWAV_LEVELS) -> bytes:
+    """Single-byte-per-bin amplitude (used for PWV3 detail waveform)."""
     if n_bins <= 0 or len(samples) == 0:
         return bytes(n_bins)
     chunks = np.array_split(np.abs(samples), n_bins)
     peaks = np.array([float(c.max()) if len(c) else 0.0 for c in chunks])
-    peak_max = float(peaks.max()) if peaks.size else 0.0
-    if peak_max <= 0:
+    return _quantise(peaks, levels).tobytes()
+
+
+def _bin_amplitudes_paired(samples: np.ndarray, n_cols: int, height_levels: int = PWAV_HEIGHT_LEVELS) -> bytes:
+    """Pair-byte-per-column quantisation for PWAV / PWV2. Returns
+    `n_cols * 2` bytes — height in even bytes (bits 0..h_bits-1),
+    intensity in odd bytes (bits 0-2).
+
+    Same gate + gamma curve as `_quantise` so silence reads as silence
+    and peaks pin to the channel max.
+    """
+    if n_cols <= 0 or samples.size == 0:
+        return bytes(n_cols * 2)
+    abs_s = np.abs(samples)
+    chunks = np.array_split(abs_s, n_cols)
+    peaks = np.array([float(c.max()) if len(c) else 0.0 for c in chunks])
+    rms = np.array([float(np.sqrt(np.mean(c.astype(np.float64) ** 2))) if len(c) else 0.0
+                    for c in chunks])
+    heights = _quantise(peaks, height_levels)
+    # Transients (peak > 1.5 × rms) get intensity 7. Heights already
+    # gated to 0 in silence — pair them: zero height → zero intensity.
+    intensities = np.where(
+        (heights > 0) & (peaks > 1.5 * np.maximum(rms, 1e-9)),
+        7, 0,
+    ).astype(np.uint8)
+    out = np.empty(n_cols * 2, dtype=np.uint8)
+    out[0::2] = heights
+    out[1::2] = intensities
+    return out.tobytes()
+
+
+def _band_envelopes(samples: np.ndarray, sr: int) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+    """Return per-sample low/mid/high envelopes (after band-pass + abs).
+    Uses a 4-pole Butterworth band-pass per band; scipy.signal.sosfilt
+    streams in a single pass so the whole-track envelope fits in RAM."""
+    from scipy.signal import butter, sosfilt
+    def env(lo: int, hi: int) -> np.ndarray:
+        sos = butter(4, [lo, hi], btype="band", fs=sr, output="sos")
+        return np.abs(sosfilt(sos, samples)).astype(np.float32)
+    return env(*BAND_LOW), env(*BAND_MID), env(*BAND_HIGH)
+
+
+def _bin_to_bytes(values: np.ndarray, n_bins: int, levels: int = 255) -> bytes:
+    """Bin `values` into `n_bins`, take per-bin max, normalise to `levels`
+    via the shared gate+gamma curve (matches what real rekordbox writes:
+    silence at zero, mids pulled toward max)."""
+    if n_bins <= 0 or values.size == 0:
         return bytes(n_bins)
-    q = np.clip(np.round(peaks / peak_max * levels), 0, levels).astype(np.uint8)
-    return q.tobytes()
+    chunks = np.array_split(values, n_bins)
+    peaks = np.array([float(c.max()) if len(c) else 0.0 for c in chunks])
+    return _quantise(peaks, levels).tobytes()
 
 
 def extract_waveform_envelope(path: Path) -> dict[str, bytes]:
@@ -204,7 +272,15 @@ def _project_beat_grid(beats: list[float], bpm: float, duration_s: float) -> lis
     return grid
 
 
-CACHE_VERSION = 1
+CACHE_VERSION = 2
+
+# 3-band waveform split. Frequencies tuned to roughly match what
+# rekordbox shows on the CDJ-3000 strip: bass (kick + sub), mid (snare,
+# vocals, chords), high (hats, cymbals).
+BAND_LOW = (20, 200)
+BAND_MID = (200, 2000)
+BAND_HIGH = (2000, 15000)
+PWV6_COLS = 1200             # 3-band preview width (fixed by spec)
 
 
 def _sanitise_isrc(isrc: str) -> str:
@@ -232,7 +308,16 @@ def save_analysis(cache_dir: Path, isrc: str, analysis: dict) -> Path | None:
         "bpm": analysis.get("bpm"),
         "beats": analysis.get("beats", []),
         "preview_b64": b64encode(analysis.get("preview") or b"").decode("ascii"),
-        "scroll_b64": b64encode(analysis.get("scroll") or b"").decode("ascii"),
+        "tiny_b64":    b64encode(analysis.get("tiny")    or b"").decode("ascii"),
+        "scroll_b64":  b64encode(analysis.get("scroll")  or b"").decode("ascii"),
+        # 3-band envelopes (cache v2). Kept separate from the mono streams
+        # so callers can ask for whichever they need without re-decoding.
+        "preview_low_b64":  b64encode(analysis.get("preview_low")  or b"").decode("ascii"),
+        "preview_mid_b64":  b64encode(analysis.get("preview_mid")  or b"").decode("ascii"),
+        "preview_high_b64": b64encode(analysis.get("preview_high") or b"").decode("ascii"),
+        "scroll_low_b64":   b64encode(analysis.get("scroll_low")   or b"").decode("ascii"),
+        "scroll_mid_b64":   b64encode(analysis.get("scroll_mid")   or b"").decode("ascii"),
+        "scroll_high_b64":  b64encode(analysis.get("scroll_high")  or b"").decode("ascii"),
     }
     target.write_text(json.dumps(payload, separators=(",", ":")))
     return target
@@ -249,10 +334,16 @@ def load_analysis(cache_dir: Path, isrc: str) -> dict | None:
             return None
         return {
             "bpm": d.get("bpm"),
-            # JSON tuples come back as lists; PQTZ writer accepts either.
             "beats": [tuple(b) for b in d.get("beats", [])],
-            "preview": b64decode(d.get("preview_b64") or ""),
-            "scroll": b64decode(d.get("scroll_b64") or ""),
+            "preview":      b64decode(d.get("preview_b64")      or ""),
+            "tiny":         b64decode(d.get("tiny_b64")         or ""),
+            "scroll":       b64decode(d.get("scroll_b64")       or ""),
+            "preview_low":  b64decode(d.get("preview_low_b64")  or ""),
+            "preview_mid":  b64decode(d.get("preview_mid_b64")  or ""),
+            "preview_high": b64decode(d.get("preview_high_b64") or ""),
+            "scroll_low":   b64decode(d.get("scroll_low_b64")   or ""),
+            "scroll_mid":   b64decode(d.get("scroll_mid_b64")   or ""),
+            "scroll_high":  b64decode(d.get("scroll_high_b64")  or ""),
         }
     except (json.JSONDecodeError, OSError, ValueError) as e:
         logger.warning("analysis cache read failed for {}: {}", isrc, e)
@@ -262,18 +353,29 @@ def load_analysis(cache_dir: Path, isrc: str) -> dict | None:
 def analyze_audio(path: Path) -> dict:
     """One-pass decode + analysis. Returns:
         {
-            "bpm":     float | None,   # median-of-intervals tempo
-            "beats":   list[(beat_n, tempo_x100, time_ms)],  # for PQTZ
-            "preview": bytes,          # for PWAV (400-byte amplitude strip)
-            "scroll":  bytes,          # for PWV3 (~150 cols/sec)
+          "bpm":      float | None,
+          "beats":    list[(beat_n, tempo_x100, time_ms)],  # for PQTZ
+          "preview":  bytes,          # 400 cols, 5-bit height        (PWAV)
+          "scroll":   bytes,          # ~150 cols/sec, 5-bit          (PWV3)
+          # 3-band envelopes:
+          "preview_low":   bytes,     # PWV6_COLS × 1B (0-255)
+          "preview_mid":   bytes,     # PWV6_COLS × 1B
+          "preview_high":  bytes,     # PWV6_COLS × 1B
+          "scroll_low":    bytes,     # ~150 cols/sec × 1B
+          "scroll_mid":    bytes,
+          "scroll_high":   bytes,
         }
     """
     samples = _decode_pcm(path)
-    out: dict = {"bpm": None, "beats": [], "preview": b"", "scroll": b""}
+    out: dict = {
+        "bpm": None, "beats": [],
+        "preview": b"", "tiny": b"", "scroll": b"",
+        "preview_low": b"", "preview_mid": b"", "preview_high": b"",
+        "scroll_low": b"", "scroll_mid": b"", "scroll_high": b"",
+    }
     if samples.size == 0:
         return out
 
-    # Beat onsets.
     beat_times = _detect_beats_in_samples(samples)
     if len(beat_times) >= MIN_BEATS:
         intervals = pl.Series(
@@ -285,9 +387,21 @@ def analyze_audio(path: Path) -> dict:
             duration_s = samples.size / SAMPLE_RATE
             out["beats"] = _project_beat_grid(beat_times, out["bpm"], duration_s)
 
-    # Waveform envelopes.
     duration = samples.size / SAMPLE_RATE
     scroll_cols = max(1, int(round(duration * SCROLL_FPS)))
-    out["preview"] = _bin_amplitudes(samples, PREVIEW_COLS)
-    out["scroll"] = _bin_amplitudes(samples, scroll_cols)
+    # PWAV / PWV2 use paired bytes (height + intensity); PWV3 is 1B/col.
+    out["preview"] = _bin_amplitudes_paired(samples, PWAV_COLS, PWAV_HEIGHT_LEVELS)
+    out["tiny"]    = _bin_amplitudes_paired(samples, PWV2_COLS, PWV2_HEIGHT_LEVELS)
+    out["scroll"]  = _bin_amplitudes(samples, scroll_cols)
+
+    # 3-band envelopes for PWV4/5/6/7 — band-pass with scipy, then bin
+    # to (a) 1200 cols for PWV4/PWV6 previews and (b) the same scroll
+    # column count as PWV3 for PWV5/PWV7.
+    lo_env, mid_env, hi_env = _band_envelopes(samples, SAMPLE_RATE)
+    out["preview_low"]  = _bin_to_bytes(lo_env,  PWV6_COLS)
+    out["preview_mid"]  = _bin_to_bytes(mid_env, PWV6_COLS)
+    out["preview_high"] = _bin_to_bytes(hi_env,  PWV6_COLS)
+    out["scroll_low"]   = _bin_to_bytes(lo_env,  scroll_cols)
+    out["scroll_mid"]   = _bin_to_bytes(mid_env, scroll_cols)
+    out["scroll_high"]  = _bin_to_bytes(hi_env,  scroll_cols)
     return out

maple-py/src/maple/anlz.py

@@ -83,39 +83,47 @@ def _section_pqtz(beats: list[tuple[int, int, int]]) -> bytes:
 
 
 def _section_pwav(preview: bytes) -> bytes:
-    """PWAV: WavePreviewTag — 400-byte blue/white touch-strip preview.
-    Each byte = (white_flag << 5) | height (bits 0-4)."""
+    """PWAV: WavePreviewTag — 400-byte touch-strip preview.
+
+    Per Beat Link's WaveformPreview reader, PWAV stores 200 col-pairs:
+        byte 2N   : bits 0-4 = height (0-31)
+        byte 2N+1 : bits 0-2 = intensity (0-7); ≥5 renders bright cyan
+    `preview` is expected to already be in that format (`maple.analyze.
+    _bin_amplitudes_paired`).
+    """
     fourcc = b"PWAV"
-    len_header = 20            # prelude(12) + len_data(4) + _unnamed1(4)
+    len_header = 20
     len_tag = len_header + len(preview)
     return fourcc + pack(">IIII", len_header, len_tag, len(preview), 0x00010000) + preview
 
 
 def _section_pwv2(tiny: bytes) -> bytes:
-    """PWV2: 100-byte "tiny" overview waveform. Same byte encoding as PWAV,
-    just at a coarser resolution. Real Q1.1 export has this section."""
+    """PWV2: 100-byte tiny preview. Same 2-bytes-per-col layout as PWAV
+    but with **4-bit** height (bits 0-3, 0-15 range) per spec."""
     fourcc = b"PWV2"
-    len_header = 20            # prelude(12) + len_data(4) + _unnamed1(4)
+    len_header = 20
     len_tag = len_header + len(tiny)
     return fourcc + pack(">IIII", len_header, len_tag, len(tiny), 0x00010000) + tiny
 
 
 def _section_pcob(cue_list_type: int) -> bytes:
     """PCOB: CueTag — cue/loop list. Real exports always include both
-    memory-cue (type=0) and hot-cue (type=1) sections, even when both are
-    empty. Body: u4 type, 2 bytes unnamed, u2 num_cues, u4 memory_count."""
+    sections in a fixed order — hot-cues (type=1) FIRST, then memory-cues
+    (type=0) — and use `0xffffffff` as the memory_count sentinel for
+    empty lists. Body: u4 type, 2 bytes unnamed, u2 num_cues, u4
+    memory_count. Verified against data/q1-rekordbox/.../ANLZ0000.DAT
+    byte-for-byte."""
     fourcc = b"PCOB"
     len_header = 24            # prelude(12) + body fixed prefix(12)
-    body = pack(">I", cue_list_type) + b"\x00\x00" + pack(">HI", 0, 0)
+    body = pack(">I", cue_list_type) + b"\x00\x00" + pack(">HI", 0, 0xFFFFFFFF)
     len_tag = 12 + len(body)
     return fourcc + pack(">II", len_header, len_tag) + body
 
 
 def _section_pwv3(scroll: bytes) -> bytes:
-    """PWV3: WaveScrollTag — full-track mono scroll waveform.
-
-    NOT written into ANLZ0000.DAT (real rekordbox places PWV3 in .EXT, not
-    .DAT). Kept here for reference / future .EXT generation."""
+    """PWV3: WaveScrollTag — full-track mono scroll waveform. Written
+    into ANLZ0000.EXT (real rekordbox puts PWV3 there, not in .DAT).
+    The loaded-deck scroll display reads from here."""
     fourcc = b"PWV3"
     len_header = 24
     len_entry_bytes = 1
@@ -124,21 +132,233 @@ def _section_pwv3(scroll: bytes) -> bytes:
     return fourcc + pack(">IIIII", len_header, len_tag, len_entry_bytes, len_entries, 0x00960000) + scroll
 
 
-def _bin_tiny_from_preview(preview: bytes, cols: int = 100) -> bytes:
-    """Derive the PWV2 tiny envelope from PWAV's 400-byte preview by
-    re-binning groups of 4 columns to their per-bin max."""
-    if not preview:
-        return b""
-    chunk = max(1, len(preview) // cols)
-    out = bytearray(cols)
-    for i in range(cols):
-        slc = preview[i * chunk : (i + 1) * chunk] if i < cols - 1 else preview[i * chunk:]
-        if slc:
-            # Per-byte max ignoring the white-flag bit, then put it back at
-            # the corresponding compressed amplitude.
-            heights = [b & 0x1F for b in slc]
-            out[i] = max(heights) & 0x1F
-    return bytes(out)
+def _section_pco2(cue_list_type: int) -> bytes:
+    """PCO2: ExtendedCueTag — empty extended cue list. Real exports ship
+    both type=1 (hot) and type=0 (memory) variants in .EXT alongside the
+    PCOB sections. Body: u4 type, u4 reserved."""
+    fourcc = b"PCO2"
+    len_header = 20            # prelude(12) + body fixed prefix(8)
+    body = pack(">II", cue_list_type, 0)
+    len_tag = 12 + len(body)
+    return fourcc + pack(">II", len_header, len_tag) + body
+
+
+# --------------------------------------------------------------------------- #
+# Colour + 3-band waveform sections (live in .EXT / .2EX)                     #
+# --------------------------------------------------------------------------- #
+
+
+def _section_pwv4(low: bytes, mid: bytes, high: bytes) -> bytes:
+    """PWV4: WaveColorPreviewTag — fixed-width colour preview, 1200 cols ×
+    6 bytes/col. Per beatlink/deepsymmetry docs, the 6 bytes per column
+    interleave 6 "stripes" rendered top-to-bottom:
+
+        byte 0 : whiteness 1   (transient highlight)
+        byte 1 : whiteness 2   (transient highlight)
+        byte 2 : energy lower-half of the frequency range
+        byte 3 : energy lower-third
+        byte 4 : energy middle
+        byte 5 : energy top
+
+    The exact whiteness derivation isn't documented. We synthesise it as
+    the column's overall amplitude (peak of all bands), which gives the
+    same shape as the existing PWAV "intense column" highlight.
+    """
+    fourcc = b"PWV4"
+    len_header = 24
+    n = min(len(low), len(mid), len(high))
+    # Per-column max across bands → whiteness stand-in
+    lo  = bytearray(low[:n])
+    mid_arr = bytearray(mid[:n])
+    hi  = bytearray(high[:n])
+    overall = bytearray(max(lo[i], mid_arr[i], hi[i]) for i in range(n))
+    body = bytearray(n * 6)
+    for i in range(n):
+        body[i * 6 + 0] = overall[i]    # whiteness 1
+        body[i * 6 + 1] = overall[i]    # whiteness 2 (same — we don't have a separate channel)
+        body[i * 6 + 2] = lo[i]         # bottom-half energy ≈ low band
+        body[i * 6 + 3] = lo[i]         # bottom-third energy ≈ low band (closer to bottom)
+        body[i * 6 + 4] = mid_arr[i]    # middle energy ≈ mid band
+        body[i * 6 + 5] = hi[i]         # top energy ≈ high band
+    len_entry_bytes = 6
+    len_entries = n
+    len_tag = len_header + len(body)
+    # Real export has `unknown = 0` here.
+    return fourcc + pack(">IIIII", len_header, len_tag, len_entry_bytes, len_entries, 0x00000000) + bytes(body)
+
+
+def _section_pwv5(low: bytes, mid: bytes, high: bytes) -> bytes:
+    """PWV5: WaveColorScrollTag — colour scroll, 2 bytes/col packed as
+
+        fedcba9876543210
+        RRRGGGBBBHHHHH--
+
+    (3 bits R, 3 bits G, 3 bits B, 5 bits height, 2 unused).
+    Map our (low, mid, high) band amplitudes (0-255) to (R, G, B) at
+    3-bit each. Height = peak across bands at 5-bit. Result: warm reds
+    for kick-heavy columns, blues for hat-heavy, etc. — matches how
+    rekordbox renders the NXS2 color waveform from low/mid/high content.
+    """
+    fourcc = b"PWV5"
+    len_header = 24
+    n = min(len(low), len(mid), len(high))
+    body = bytearray(n * 2)
+    for i in range(n):
+        # Map 0-255 → 0-7 (3-bit per channel). Low band → red, mid → green,
+        # high → blue (matches the "RGB ≈ low/mid/high" convention rekordbox
+        # uses when reconstructing the NXS2 strip).
+        r = low[i]  >> 5
+        g = mid[i]  >> 5
+        b = high[i] >> 5
+        h_in = max(low[i], mid[i], high[i])
+        h = h_in >> 3  # 0-255 → 0-31
+        word = (r & 0x7) << 13 | (g & 0x7) << 10 | (b & 0x7) << 7 | (h & 0x1F) << 2
+        body[i * 2]     = (word >> 8) & 0xFF
+        body[i * 2 + 1] = word & 0xFF
+    len_entry_bytes = 2
+    len_entries = n
+    len_tag = len_header + len(body)
+    return fourcc + pack(">IIIII", len_header, len_tag, len_entry_bytes, len_entries, 0x00960305) + bytes(body)
+
+
+def _section_pwv6(low: bytes, mid: bytes, high: bytes) -> bytes:
+    """PWV6: Wave3bandPreviewTag (CDJ-3000). 3 bytes/col, order **mid,
+    high, low** per the docs (NOT low-mid-high). Fixed 1200 cols."""
+    fourcc = b"PWV6"
+    len_header = 20
+    n = min(len(low), len(mid), len(high))
+    body = bytearray(n * 3)
+    for i in range(n):
+        body[i * 3 + 0] = mid[i]
+        body[i * 3 + 1] = high[i]
+        body[i * 3 + 2] = low[i]
+    len_entry_bytes = 3
+    len_entries = n
+    len_tag = len_header + len(body)
+    return fourcc + pack(">IIII", len_header, len_tag, len_entry_bytes, len_entries) + bytes(body)
+
+
+def _section_pwv7(low: bytes, mid: bytes, high: bytes) -> bytes:
+    """PWV7: Wave3bandScrollTag (CDJ-3000). Same 3-byte mid/high/low
+    encoding as PWV6 but variable length at ~150 cols/sec."""
+    fourcc = b"PWV7"
+    len_header = 24
+    n = min(len(low), len(mid), len(high))
+    body = bytearray(n * 3)
+    for i in range(n):
+        body[i * 3 + 0] = mid[i]
+        body[i * 3 + 1] = high[i]
+        body[i * 3 + 2] = low[i]
+    len_entry_bytes = 3
+    len_entries = n
+    len_tag = len_header + len(body)
+    return fourcc + pack(">IIIII", len_header, len_tag, len_entry_bytes, len_entries, 0x00960000) + bytes(body)
+
+
+def _section_pqt2(beats: list[tuple[int, int, int]]) -> bytes:
+    """PQT2: extended beat grid (CDJ-3000+). Layout is only partially
+    documented; the format isn't in the kaitai schema or the deepsymmetry
+    spec page. The hex dump of a real Q1 PQT2 shows a 56-byte header
+    carrying num_beats, first_beat_ms, mid_ms, end_ms (matching the
+    corresponding PQTZ values), then per-beat 2-byte entries.
+
+    We write a placeholder: header fields mirror PQTZ where we know
+    them, the rest zero-filled. Per-beat entries are 2 bytes of zero
+    each (encoding unknown). Empirically the CDJ-2000NXS2+ uses the
+    presence of PQT2 — independently of its content — as a gate for the
+    deck-side waveform / BPM display.
+    """
+    fourcc = b"PQT2"
+    len_header = 56
+    num_beats = len(beats)
+    first_beat_ms = beats[0][2] if beats else 0
+    if num_beats >= 2:
+        interval = beats[-1][2] - beats[-2][2]
+        end_ms = beats[-1][2] + interval
+    else:
+        end_ms = first_beat_ms
+    mid_ms = end_ms // 2
+    # 44 bytes of body header (11 × u4). First few fields known from the
+    # real Q1 PQT2 dump; later u4s look like an analyser-timestamp that
+    # we can't reproduce, so we zero them.
+    body_header = pack(
+        ">11I",
+        0,                  # unknown1
+        0x01000002,          # unknown2 — observed constant in real exports
+        0,                  # unknown3
+        0x00013200,          # unknown4 — observed constant
+        first_beat_ms,
+        mid_ms,
+        end_ms,
+        num_beats,
+        0,                  # unknown9 (date/timestamp; not reproduced)
+        0,                  # unknown10
+        0,                  # unknown11
+    )
+    entries = b"\x00\x00" * num_beats
+    body = body_header + entries
+    len_tag = 12 + len(body)
+    return fourcc + pack(">II", len_header, len_tag) + body
+
+
+def _section_pwvc() -> bytes:
+    """PWVC: 20-byte tag at the tail of `.2EX`. Undocumented. Real exports
+    always include it; we emit a zero-filled placeholder of the expected
+    length so the file ends with the same section list."""
+    fourcc = b"PWVC"
+    len_header = 14
+    body = b"\x00" * 8
+    len_tag = 12 + len(body)
+    return fourcc + pack(">II", len_header, len_tag) + body
+
+
+# XOR mask used by rekordbox to obfuscate the PSSI body. Mask is offset
+# per byte by the (unmasked) `len_entries` value before XOR'ing in.
+_PSSI_MASK = bytes.fromhex("CBE1EEFAE5EEADEEE9D2E9EBE1E9F3E8E9F4E1")
+
+
+def _section_pssi(num_beats: int) -> bytes:
+    """PSSI: SongStructureTag — phrase analysis. We emit an empty (0
+    phrases) placeholder so the CDJ-3000 sees a valid PSSI and doesn't
+    blank its phrase indicators. Mood=2 (mid), bank=0 (default).
+
+    Per spec, bytes 0x12 onwards are XOR-masked with `_PSSI_MASK[i] +
+    len_entries` (mod 256). Since `len_entries == 0`, the mask itself is
+    the XOR key.
+    """
+    fourcc = b"PSSI"
+    len_header = 32
+    # Unmasked body fields:
+    len_entry_bytes = 0x18
+    len_entries = 0           # no phrases
+    mood = 2                  # mid
+    unknown_u4 = 0
+    end_beat = num_beats      # final phrase end beat = last beat in PQTZ
+    unknown2_u4 = 0
+    bank = 0                  # default
+    unknown3 = 0
+
+    unmasked = pack(">I", len_entry_bytes)  # offset 0x0c-0x0f (NOT masked per spec)
+    masked_block = (
+        pack(">H", len_entries)             # 0x10-0x11
+        + pack(">H", mood)                  # 0x12-0x13
+        + pack(">I", unknown_u4)            # 0x14-0x17
+        + pack(">H", end_beat)              # 0x18-0x19
+        + pack(">I", unknown2_u4)           # 0x1a-0x1d
+        + pack(">B", bank)                  # 0x1e
+        + pack(">B", unknown3)              # 0x1f
+    )
+    # Apply mask: spec says "all bytes from offset 0x12 onwards" — that's
+    # the 2-byte mood field onward. Skip the first 2 bytes of masked_block
+    # (which are len_entries at 0x10).
+    masked_out = bytearray(masked_block[:2])  # len_entries unmasked
+    for i, b in enumerate(masked_block[2:]):
+        masked_out.append(b ^ ((_PSSI_MASK[i % len(_PSSI_MASK)] + len_entries) & 0xFF))
+
+    body = unmasked + bytes(masked_out)
+    # No entries to append (len_entries=0).
+    len_tag = 12 + len(body)
+    return fourcc + pack(">II", len_header, len_tag) + body
 
 
 def _pmai_header(file_len: int) -> bytes:
@@ -158,22 +378,66 @@ def build_dat(rb_path: str, analysis: dict | None) -> bytes:
         if analysis.get("beats"):
             sections.append(_section_pqtz(analysis["beats"]))
         preview = analysis.get("preview") or b""
+        tiny    = analysis.get("tiny") or b""
         if preview:
             sections.append(_section_pwav(preview))
-            sections.append(_section_pwv2(_bin_tiny_from_preview(preview)))
+        if tiny:
+            sections.append(_section_pwv2(tiny))
+    sections.append(_section_pcob(1))   # hot cues (empty) — real writes hot-cues first
     sections.append(_section_pcob(0))   # memory cues (empty)
-    sections.append(_section_pcob(1))   # hot cues (empty)
     body = b"".join(sections)
     return _pmai_header(ANLZ_HEADER_LEN + len(body)) + body
 
 
-def build_ext(rb_path: str) -> bytes:
-    """Build a minimal ANLZ0000.EXT — PPTH-only.
+def build_ext(rb_path: str, analysis: dict | None = None) -> bytes:
+    """Build ANLZ0000.EXT — extended analysis data the loaded-deck reads.
 
-    Real rekordbox .EXT carries color waveforms and song-structure
-    sections; we don't generate those yet. A bare .EXT file is enough for
-    firmware that opens it as part of the standard track-load flow."""
-    body = _section_pth(rb_path)
+    Section order matches the real Q1 reference:
+        PPTH, PWV3 (mono scroll), PCOB×2, PCO2×2,
+        [PQT2 — skipped, format undocumented],
+        PWV5 (colour scroll), PWV4 (colour preview), PSSI (empty).
+    """
+    sections = [_section_pth(rb_path)]
+    if analysis and analysis.get("scroll"):
+        sections.append(_section_pwv3(analysis["scroll"]))
+    sections.append(_section_pcob(1))
+    sections.append(_section_pcob(0))
+    sections.append(_section_pco2(1))
+    sections.append(_section_pco2(0))
+    if analysis and analysis.get("beats"):
+        # PQT2 between PCO2 and PWV5 — matches real-export section order
+        # and (empirically) gates the CDJ's deck-side waveform/BPM display.
+        sections.append(_section_pqt2(analysis["beats"]))
+    if analysis:
+        s_lo, s_md, s_hi = analysis.get("scroll_low") or b"", analysis.get("scroll_mid") or b"", analysis.get("scroll_high") or b""
+        if s_lo and s_md and s_hi:
+            sections.append(_section_pwv5(s_lo, s_md, s_hi))
+        p_lo, p_md, p_hi = analysis.get("preview_low") or b"", analysis.get("preview_mid") or b"", analysis.get("preview_high") or b""
+        if p_lo and p_md and p_hi:
+            sections.append(_section_pwv4(p_lo, p_md, p_hi))
+        num_beats = len(analysis.get("beats") or [])
+        if num_beats:
+            sections.append(_section_pssi(num_beats))
+    body = b"".join(sections)
+    return _pmai_header(ANLZ_HEADER_LEN + len(body)) + body
+
+
+def build_2ex(rb_path: str, analysis: dict | None = None) -> bytes:
+    """Build ANLZ0000.2EX — CDJ-3000 3-band waveforms.
+
+    Section order matches the real Q1 reference:
+        PPTH, PWV7 (3-band scroll), PWV6 (3-band preview), PWVC (skipped).
+    """
+    sections = [_section_pth(rb_path)]
+    if analysis:
+        s_lo, s_md, s_hi = analysis.get("scroll_low") or b"", analysis.get("scroll_mid") or b"", analysis.get("scroll_high") or b""
+        if s_lo and s_md and s_hi:
+            sections.append(_section_pwv7(s_lo, s_md, s_hi))
+        p_lo, p_md, p_hi = analysis.get("preview_low") or b"", analysis.get("preview_mid") or b"", analysis.get("preview_high") or b""
+        if p_lo and p_md and p_hi:
+            sections.append(_section_pwv6(p_lo, p_md, p_hi))
+    sections.append(_section_pwvc())   # 20-byte trailer real exports always include
+    body = b"".join(sections)
     return _pmai_header(ANLZ_HEADER_LEN + len(body)) + body
 
 
@@ -252,9 +516,11 @@ def write_anlz(
                 logger.warning("could not write analysis cache for {}: {}", source_audio.name, e)
 
     dest.write_bytes(build_dat(rb_path, analysis))
-    # .EXT sister file. Newer firmware (esp. CDJ-3000) opens this during the
-    # track-load flow; absence of the file has been observed to silently
-    # disable the loaded-deck BPM/waveform display even when .DAT is intact.
     ext_path = dest.with_suffix(".EXT")
-    ext_path.write_bytes(build_ext(rb_path))
+    ext_path.write_bytes(build_ext(rb_path, analysis))
+    # .2EX carries the CDJ-3000's 3-band waveforms (PWV6/PWV7). Older
+    # firmware ignores the file; CDJ-3000 reads it for the three-band
+    # strip display.
+    two_ex_path = dest.with_suffix(".2EX")
+    two_ex_path.write_bytes(build_2ex(rb_path, analysis))
     return analysis

maple-py/src/maple/cli.py

@@ -5,13 +5,14 @@ import re
 import sys
 import time
 from collections import defaultdict
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import date
 from pathlib import Path
 
 from loguru import logger
 
-from maple.anlz import write_anlz
+from maple.analyze import analyze_audio, cache_path_for, save_analysis
+from maple.anlz import _isrc_for, write_anlz
 from maple.artwork import _bucket as _artwork_bucket
 from maple.artwork import extract_artwork
 from maple.convert import convert_flac_to_mp3, verify_duration
@@ -124,8 +125,8 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> tuple
         converted = convert_flac_to_mp3(src, dst)
         return src, dst, converted, time.perf_counter() - t0
 
+    workers = max(1, os.cpu_count() or 1)
     if flac_jobs:
-        workers = max(1, os.cpu_count() or 1)
         logger.info("converting {} flac(s) → mp3 v0 (workers={})", len(flac_jobs), workers)
         t = time.perf_counter()
         n_converted = 0
@@ -141,6 +142,69 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> tuple
             n_converted, len(flac_jobs) - n_converted, time.perf_counter() - t,
         )
 
+    # Parallel pre-analysis. The per-track build loop below is sequential
+    # (the pdb / artwork-id assignments depend on iteration order), but
+    # `analyze_audio` is the slow step and trivially parallel: ffmpeg +
+    # aubio + scipy each release the GIL, so threads scale linearly with
+    # cores. After this pass every track has its ISRC-keyed analysis
+    # cached, so write_anlz in the build loop is a cache hit.
+    def _dest_for(p: Path) -> Path:
+        m = initial_meta[p]
+        if p.suffix.lower() == ".flac":
+            dest_abs, _ = staged_path(usb_root, m.artist, m.album, p.with_suffix(".mp3").name)
+        else:
+            dest_abs, _ = staged_path(usb_root, m.artist, m.album, p.name)
+        return dest_abs
+
+    # Make sure non-FLAC sources are staged to data/usb first so the
+    # analysis pass operates on the file the CDJ will actually read.
+    for p in audio_files:
+        if p.suffix.lower() != ".flac":
+            m = initial_meta[p]
+            stage_audio(p, usb_root, m.artist, m.album)
+
+    def _ensure_analysis_cache(p: Path) -> bool:
+        """Populate `<cache>/<isrc>.json` if missing. Returns True iff a
+        fresh analyse_audio actually ran (i.e. cache was cold)."""
+        dest = _dest_for(p)
+        if not dest.exists():
+            return False
+        isrc = _isrc_for(dest)
+        if not isrc:
+            return False
+        cp = cache_path_for(cache_dir, isrc)
+        if cp is None or cp.exists():
+            return False
+        try:
+            a = analyze_audio(dest)
+        except Exception as e:
+            logger.warning("analyze failed for {}: {}", dest.name, e)
+            return False
+        if a:
+            save_analysis(cache_dir, isrc, a)
+            return True
+        return False
+
+    pre_t0 = time.perf_counter()
+    logger.info("pre-analysing {} tracks (workers={})...", len(audio_files), workers)
+    new_count = 0
+    done = 0
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        futures = {ex.submit(_ensure_analysis_cache, p): p for p in audio_files}
+        for fut in as_completed(futures):
+            done += 1
+            try:
+                if fut.result():
+                    new_count += 1
+            except Exception as e:
+                logger.warning("pre-analysis error: {}", e)
+            if done % 8 == 0 or done == len(audio_files):
+                logger.info("  [{}/{}] cache misses analysed: {}", done, len(audio_files), new_count)
+    logger.info(
+        "pre-analysis: {} new in {:.1f}s ({} already cached)",
+        new_count, time.perf_counter() - pre_t0, len(audio_files) - new_count,
+    )
+
     logger.info("building track table ({} files)", len(audio_files))
     build_t0 = time.perf_counter()
     validation_issues: list[TrackIssue] = []
@@ -217,13 +281,23 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> tuple
         # path keyed off the track id even if we don't write the .DAT.
         import zlib
         h = zlib.crc32(file_path.encode()) & 0xFFFFFFFF
-        # rekordbox's USBANLZ P-bucket is 0..127 (P00..P7F) and the 8-hex id is an
-        # opaque master.db value -- not reproducible without its DB (FORMAT_SPEC
-        # §7), so maple derives both from a stable crc32(path). Mask the bucket to
-        # 0x7F so it has rekordbox's P00..P7F shape; the path used here is the same
-        # one written to the pdb row, so it's self-consistent by construction.
-        bucket = f"P{(h >> 20) & 0x7F:02X}"
-        analyze_id = f"{h & 0xFFFFFFFF:08X}"
+        # rekordbox writes 3-digit zero-padded buckets (e.g. P020 not P20) +
+        # 8-hex-digit ids. The bucket value range observed in real exports
+        # is 0..127 (P000..P07F) — wider masks here haven't shown up in any
+        # reference export and may push the CDJ outside whatever lookup
+        # range its firmware accepts. The 8-hex id is an opaque master.db
+        # value; we derive a stable-but-small value from crc32(path) capped
+        # to 20 bits so it lives in the same numeric range as real-export
+        # ids (Q1's id is 0x0002E10A ≈ 190K).
+        bucket = f"P{(h >> 20) & 0x7F:03X}"
+        analyze_id = f"{h & 0xFFFFF:08X}"
+        # DEBUG: force Q1.1 to use the exact ANLZ path the real rekordbox
+        # export uses — settles whether the CDJ's path-template parser
+        # cares about the bucket/id pattern.
+        if file_path == "/Contents/Basic Channel/Q 1.1/02 Q1.1 II.mp3":
+            bucket = "P020"
+            analyze_id = "0002E10A"
+            logger.warning("Q1.1 path forced to real rekordbox bucket: {}/{}", bucket, analyze_id)
         analyze_path = f"/PIONEER/USBANLZ/{bucket}/{analyze_id}/ANLZ0000.DAT"
         anlz_abs = (usb_root / analyze_path.lstrip("/")).resolve()
         # `_analysis/` sits at the input collection root; ISRC-keyed cache
@@ -234,8 +308,9 @@ def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> tuple
             analysis_cache_dir=input_dir / "_analysis",
         )
         written.add(anlz_abs)
-        # .EXT sister file is written automatically next to the .DAT.
+        # .EXT and .2EX sister files are written automatically next to .DAT.
         written.add(anlz_abs.with_suffix(".EXT"))
+        written.add(anlz_abs.with_suffix(".2EX"))
         # Prefer the analytically detected BPM over the file tag when we have one.
         # CDJ browse panel reads pdb.tempo; loaded-deck readout reads the PQTZ
         # we just wrote — keeping the two in sync avoids the "120.0 in browse,

maple-py/src/maple/pdb/rows.py

@@ -17,7 +17,13 @@ TRACK_HEADER_LEN = TRACK_OFS_STRINGS_START + 21 * 2
 # earlier output (data/usb), not a real export. Default to this install's real
 # value; override per-install if deck testing shows it matters. Players treat
 # the whole blob as opaque.
-TRACK_OPAQUE_SEED = 0x21B4A7AB
+# Per-export "seed" stored at offset 0x18 of every TrackRow. Real rekordbox
+# files carry the LE u4 0xd6d3a567 here (parses as opaque_seed_lo=0xa567,
+# opaque_seed_hi=0xd6d3). Maple previously used 0x21B4A7AB which the kaitai
+# would parse as different seed_lo/hi values — the difference may be benign
+# for the CDJ's analyse-path lookup but matches every real reference export
+# we have, so we mirror it.
+TRACK_OPAQUE_SEED = 0xD6D3A567
 
 # Range maple maps its stand-in opaque_track_id_lo (row 0x14) into, so it has the
 # shape of a real rekordbox Content ID. The real AceMo lo values all fall in this