kaitai py

maple-py/compare_pdb.py

@@ -0,0 +1,57 @@
+"""Parse two .pdb files with the kaitai schema and summarise differences."""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent / "src"))
+from io import BytesIO
+from kaitaistruct import KaitaiStream
+from kaitai.rekordbox_pdb import RekordboxPdb
+
+
+PAGE_TYPE_NAMES = {t.value: t.name for t in RekordboxPdb.PageType}
+
+
+def summarise(path: Path):
+    data = path.read_bytes()
+    pdb = RekordboxPdb(is_ext=False, _io=KaitaiStream(BytesIO(data)))
+    print(f"\n=== {path} ({path.stat().st_size} bytes) ===")
+    print(f"page_size={pdb.len_page} num_tables={pdb.num_tables} next_unused_page={pdb.next_unused_page} sequence={pdb.sequence}")
+    for tbl in pdb.tables:
+        ptype = PAGE_TYPE_NAMES.get(tbl.type.value, str(tbl.type))
+        page_count = 0
+        row_count = 0
+        err = None
+        try:
+            ref = tbl.first_page
+            while True:
+                page = ref.body
+                page_count += 1
+                if page.is_data_page:
+                    for rg in page.row_groups:
+                        for rpw in rg.rows:
+                            if rpw.present:
+                                row_count += 1
+                if ref.index == tbl.last_page.index:
+                    break
+                ref = page.next_page
+                if page_count > 5000:
+                    break
+        except Exception as e:
+            err = f"{type(e).__name__}: {e}"
+        line = f"  {ptype:24s} first={tbl.first_page.index:>4} last={tbl.last_page.index:>4}  pages={page_count:>3} rows={row_count}"
+        if err:
+            line += f"  ERR={err}"
+        print(line)
+    return pdb
+
+
+def main():
+    base = Path(__file__).parent
+    ref = base / "data/MAPLE/PIONEER/rekordbox/export.pdb"
+    ours = base / "data/usb/PIONEER/rekordbox/export.pdb"
+    summarise(ref)
+    summarise(ours)
+
+
+if __name__ == "__main__":
+    main()

maple-py/pyproject.toml

@@ -8,9 +8,14 @@ authors = [
 ]
 requires-python = ">=3.14"
 dependencies = [
+    "kaitaistruct>=0.11",
     "mutagen>=1.47.0",
+    "pillow>=12.2.0",
 ]
 
+[project.scripts]
+maple = "maple.cli:main"
+
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"

maple-py/src/kaitai/rekordbox_anlz.ksy

@@ -0,0 +1,673 @@
+meta:
+  id: rekordbox_anlz
+  title: rekordbox track analysis file
+  application: rekordbox
+  file-extension:
+    - dat
+    - ext
+  license: EPL-1.0
+  endian: be
+
+doc: |
+  These files are created by rekordbox when analyzing audio tracks
+  to facilitate DJ performance. They include waveforms, beat grids
+  (information about the precise time at which each beat occurs),
+  time indices to allow efficient seeking to specific positions
+  inside variable bit-rate audio streams, and lists of memory cues
+  and loop points. They are used by Pioneer professional DJ
+  equipment.
+
+  The format has been reverse-engineered to facilitate sophisticated
+  integrations with light and laser shows, videos, and other musical
+  instruments, by supporting deep knowledge of what is playing and
+  what is coming next through monitoring the network communications
+  of the players.
+
+doc-ref: https://reverseengineering.stackexchange.com/questions/4311/help-reversing-a-edb-database-file-for-pioneers-rekordbox-software
+
+seq:
+  - id: magic
+    contents: "PMAI"
+    doc: Identifies this as an analysis file.
+  - id: len_header
+    type: u4
+    doc: |
+      The number of bytes of this header section.
+  - id: len_file
+    type: u4
+    doc: |
+       The number of bytes in the entire file.
+  - size: len_header - _io.pos
+  - id: sections
+    type: tagged_section
+    repeat: eos
+    doc: |
+      The remainder of the file is a sequence of type-tagged sections,
+      identified by a four-byte magic sequence.
+
+types:
+  tagged_section:
+    doc: |
+      A type-tagged file section, identified by a four-byte magic
+      sequence, with a header specifying its length, and whose payload
+      is determined by the type tag.
+    seq:
+      - id: fourcc
+        type: s4
+        enum: section_tags
+        doc: |
+          A tag value indicating what kind of section this is.
+      - id: len_header
+        type: u4
+        doc: |
+          The size, in bytes, of the header portion of the tag.
+      - id: len_tag
+        type: u4
+        doc: |
+          The size, in bytes, of this entire tag, counting the header.
+      - id: body
+        size: len_tag - 12
+        type:
+          switch-on: fourcc
+          cases:
+            'section_tags::cues_2': cue_extended_tag                    # PCO2
+            'section_tags::cues': cue_tag                               # PCOB
+            'section_tags::path': path_tag                              # PPTH
+            'section_tags::beat_grid': beat_grid_tag                    # PQTZ
+            'section_tags::vbr': vbr_tag                                # PVBR
+            'section_tags::wave_preview': wave_preview_tag              # PWAV
+            'section_tags::wave_tiny': wave_preview_tag                 # PWV2
+            'section_tags::wave_scroll': wave_scroll_tag                # PWV3, seen in .EXT
+            'section_tags::wave_color_preview': wave_color_preview_tag  # PWV4, in .EXT
+            'section_tags::wave_color_scroll': wave_color_scroll_tag    # PWV5, in .EXT
+            'section_tags::wave_3band_preview': wave_3band_preview_tag  # P@V6, in .2EX
+            'section_tags::wave_3band_scroll': wave_3band_scroll_tag    # PWV7, in .2EX
+            'section_tags::song_structure': song_structure_tag          # PSSI, in .EXT
+            _: unknown_tag
+    -webide-representation: '{fourcc}'
+
+
+  beat_grid_tag:
+    doc: |
+      Holds a list of all the beats found within the track, recording
+      their bar position, the time at which they occur, and the tempo
+      at that point.
+    seq:
+      - type: u4
+      - type: u4  # @flesniak says this is always 0x80000
+      - id: num_beats
+        type: u4
+        doc: |
+          The number of beat entries which follow.
+      - id: beats
+        type: beat_grid_beat
+        repeat: expr
+        repeat-expr: num_beats
+        doc: The entries of the beat grid.
+
+  beat_grid_beat:
+    doc: |
+      Describes an individual beat in a beat grid.
+    seq:
+      - id: beat_number
+        type: u2
+        doc: |
+          The position of the beat within its musical bar, where beat 1
+          is the down beat.
+      - id: tempo
+        type: u2
+        doc: |
+          The tempo at the time of this beat, in beats per minute,
+          multiplied by 100.
+      - id: time
+        type: u4
+        doc: |
+          The time, in milliseconds, at which this beat occurs when
+          the track is played at normal (100%) pitch.
+
+  cue_tag:
+    doc: |
+      Stores either a list of ordinary memory cues and loop points, or
+      a list of hot cues and loop points.
+    seq:
+      - id: type
+        type: u4
+        enum: cue_list_type
+        doc: |
+          Identifies whether this tag stores ordinary or hot cues.
+      - size: 2
+      - id: num_cues
+        type: u2
+        doc: |
+          The length of the cue list.
+      - id: memory_count
+        type: u4
+        doc: |
+          Unsure what this means.
+      - id: cues
+        type: cue_entry
+        repeat: expr
+        repeat-expr: num_cues
+
+  cue_entry:
+    doc: |
+      A cue list entry. Can either represent a memory cue or a loop.
+    seq:
+      - id: magic
+        contents: "PCPT"
+        doc: Identifies this as a cue list entry (cue point).
+      - id: len_header
+        type: u4
+      - id: len_entry
+        type: u4
+      - id: hot_cue
+        type: u4
+        doc: |
+          If zero, this is an ordinary memory cue, otherwise this a
+          hot cue with the specified number.
+      - id: status
+        type: u4
+        enum: cue_entry_status
+        doc: |
+          Indicates if this is an active loop.
+      - type: u4  # Seems to always be 0x10000
+      - id: order_first
+        type: u2
+        doc: |
+          @flesniak says: "0xffff for first cue, 0,1,3 for next"
+      - id: order_last
+        type: u2
+        doc: |
+          @flesniak says: "1,2,3 for first, second, third cue, 0xffff for last"
+      - id: type
+        type: u1
+        enum: cue_entry_type
+        doc: |
+          Indicates whether this is a memory cue or a loop.
+      - size: 3  # seems to always be 1000
+      - id: time
+        type: u4
+        doc: |
+          The position, in milliseconds, at which the cue point lies
+          in the track.
+      - id: loop_time
+        type: u4
+        doc: |
+          The position, in milliseconds, at which the player loops
+          back to the cue time if this is a loop.
+      - size: 16
+
+  cue_extended_tag:
+    doc: |
+      A variation of cue_tag which was introduced with the nxs2 line,
+      and adds descriptive names. (Still comes in two forms, either
+      holding memory cues and loop points, or holding hot cues and
+      loop points.) Also includes hot cues D through H and color assignment.
+    seq:
+      - id: type
+        type: u4
+        enum: cue_list_type
+        doc: |
+          Identifies whether this tag stores ordinary or hot cues.
+      - id: num_cues
+        type: u2
+        doc: |
+          The length of the cue comment list.
+      - size: 2
+      - id: cues
+        type: cue_extended_entry
+        repeat: expr
+        repeat-expr: num_cues
+
+  cue_extended_entry:
+    doc: |
+      A cue extended list entry. Can either describe a memory cue or a
+      loop.
+    seq:
+      - id: magic
+        contents: "PCP2"
+        doc: Identifies this as an extended cue list entry (cue point).
+      - id: len_header
+        type: u4
+      - id: len_entry
+        type: u4
+      - id: hot_cue
+        type: u4
+        doc: |
+          If zero, this is an ordinary memory cue, otherwise this a
+          hot cue with the specified number.
+      - id: type
+        type: u1
+        enum: cue_entry_type
+        doc: |
+          Indicates whether this is a regular cue point or a loop.
+      - size: 3  # seems to always be 1000
+      - id: time
+        type: u4
+        doc: |
+          The position, in milliseconds, at which the cue point lies
+          in the track.
+      - id: loop_time
+        type: u4
+        doc: |
+          The position, in milliseconds, at which the player loops
+          back to the cue time if this is a loop.
+      - id: color_id
+        type: u1
+        doc: |
+          References a row in the colors table if this is a memory cue or loop
+          and has been assigned a color.
+      - size: 7
+      - id: loop_numerator
+        type: u2
+        doc: |
+          The numerator of the loop length in beats.
+          Zero if the loop is not quantized.
+      - id: loop_denominator
+        type: u2
+        doc: |
+          The denominator of the loop length in beats.
+          Zero if the loop is not quantized.
+      - id: len_comment
+        type: u4
+        if: len_entry > 43
+      - id: comment
+        type: str
+        size: len_comment
+        encoding: utf-16be
+        doc: |
+          The comment assigned to this cue by the DJ, if any, with a trailing NUL.
+        if: len_entry > 43
+      - id: color_code
+        type: u1
+        doc: |
+          A lookup value for a color table? We use this to index to the hot cue colors shown in rekordbox.
+        if: (len_entry - len_comment) > 44
+      - id: color_red
+        type: u1
+        doc: |
+          The red component of the hot cue color to be displayed.
+        if: (len_entry - len_comment) > 45
+      - id: color_green
+        type: u1
+        doc: |
+          The green component of the hot cue color to be displayed.
+        if: (len_entry - len_comment) > 46
+      - id: color_blue
+        type: u1
+        doc: |
+          The blue component of the hot cue color to be displayed.
+        if: (len_entry - len_comment) > 47
+      - size: len_entry - 48 - len_comment  # The remainder after the color
+        if: (len_entry - len_comment) > 48
+
+  path_tag:
+    doc: |
+      Stores the file path of the audio file to which this analysis
+      applies.
+    seq:
+      - id: len_path
+        type: u4
+      - id: path
+        type: str
+        size: len_path - 2
+        encoding: utf-16be
+        if: len_path > 1
+    -webide-representation: '{path}'
+
+  vbr_tag:
+    doc: |
+      Stores an index allowing rapid seeking to particular times
+      within a variable-bitrate audio file.
+    seq:
+      - type: u4
+      - id: index
+        type: u4
+        repeat: expr
+        repeat-expr: 400
+
+  wave_preview_tag:
+    doc: |
+      Stores a waveform preview image suitable for display above
+      the touch strip for jumping to a track position.
+    seq:
+      - id: len_data
+        type: u4
+        doc: |
+          The length, in bytes, of the preview data itself. This is
+          slightly redundant because it can be computed from the
+          length of the tag.
+      - type: u4  # This seems to always have the value 0x10000
+      - id: data
+        size: len_data
+        doc: |
+          The actual bytes of the waveform preview.
+        if: _parent.len_tag > _parent.len_header
+
+  wave_scroll_tag:
+    doc: |
+      A larger waveform image suitable for scrolling along as a track
+      plays.
+    seq:
+      - id: len_entry_bytes
+        type: u4
+        doc: |
+          The size of each entry, in bytes. Seems to always be 1.
+      - id: len_entries
+        type: u4
+        doc: |
+          The number of waveform data points, each of which takes one
+          byte.
+      - type: u4  # Always 0x960000?
+      - id: entries
+        size: len_entries * len_entry_bytes
+
+  wave_color_preview_tag:
+    doc: |
+      A larger, colorful waveform preview image suitable for display
+      above the touch strip for jumping to a track position on newer
+      high-resolution players.
+    seq:
+      - id: len_entry_bytes
+        type: u4
+        doc: |
+          The size of each entry, in bytes. Seems to always be 6.
+      - id: len_entries
+        type: u4
+        doc: |
+          The number of waveform data points, each of which takes one
+          byte for each of six channels of information.
+      - type: u4
+      - id: entries
+        size: len_entries * len_entry_bytes
+
+  wave_color_scroll_tag:
+    doc: |
+      A larger, colorful waveform image suitable for scrolling along
+      as a track plays on newer high-resolution hardware. Also
+      contains a higher-resolution blue/white waveform.
+    seq:
+      - id: len_entry_bytes
+        type: u4
+        doc: |
+          The size of each entry, in bytes. Seems to always be 2.
+      - id: len_entries
+        type: u4
+        doc: |
+          The number of columns of waveform data (this matches the
+          non-color waveform length.
+      - type: u4
+      - id: entries
+        size: len_entries * len_entry_bytes
+
+  wave_3band_preview_tag:
+    doc: |
+      The minimalist CDJ-3000 waveform preview image suitable for display
+      above the touch strip for jumping to a track position.
+    seq:
+      - id: len_entry_bytes
+        type: u4
+        doc: |
+          The size of each entry, in bytes. Seems to always be 3.
+      - id: len_entries
+        type: u4
+        doc: |
+          The number of waveform data points, each of which takes one
+          byte for each of six channels of information.
+      - id: entries
+        size: len_entries * len_entry_bytes
+
+  wave_3band_scroll_tag:
+    doc: |
+      The minimalist CDJ-3000 waveform image suitable for scrolling along
+      as a track plays on newer high-resolution hardware.
+    seq:
+      - id: len_entry_bytes
+        type: u4
+        doc: |
+          The size of each entry, in bytes. Seems to always be 3.
+      - id: len_entries
+        type: u4
+        doc: |
+          The number of columns of waveform data (this matches the
+          non-color waveform length.
+      - type: u4
+      - id: entries
+        size: len_entries * len_entry_bytes
+
+  song_structure_tag:
+    doc: |
+      Stores the song structure, also known as phrases (intro, verse,
+      bridge, chorus, up, down, outro).
+    seq:
+      - id: len_entry_bytes
+        type: u4
+        doc: |
+          The size of each entry, in bytes. Seems to always be 24.
+      - id: len_entries
+        type: u2
+        doc: |
+          The number of phrases.
+      - id: body
+        type: song_structure_body
+        doc: |
+          The rest of the tag, which needs to be unmasked before it
+          can be parsed.
+        size-eos: true
+        process:  'xor(is_masked ? mask : [0])'
+    instances:
+      c:
+        value: len_entries
+      mask:
+        value: |
+          [
+            (0xCB+c).as<s1>, (0xE1+c).as<s1>, (0xEE+c).as<s1>, (0xFA+c).as<s1>, (0xE5+c).as<s1>, (0xEE+c).as<s1>, (0xAD+c).as<s1>, (0xEE+c).as<s1>,
+            (0xE9+c).as<s1>, (0xD2+c).as<s1>, (0xE9+c).as<s1>, (0xEB+c).as<s1>, (0xE1+c).as<s1>, (0xE9+c).as<s1>, (0xF3+c).as<s1>, (0xE8+c).as<s1>,
+            (0xE9+c).as<s1>, (0xF4+c).as<s1>, (0xE1+c).as<s1>
+          ].as<bytes>
+      raw_mood:
+        type: u2
+        pos: 6
+        doc: |
+          This is a way to tell whether the rest of the tag has been masked. The value is supposed
+          to range from 1 to 3, but in masked files it will be much larger.
+      is_masked:
+        value: 'raw_mood > 20'  # This is almost certainly not true for an unmasked file.
+    -webide-representation: '{body.mood}'
+
+
+  song_structure_body:
+    doc: |
+      Stores the rest of the song structure tag, which can only be
+      parsed after unmasking.
+    seq:
+      - id: mood
+        type: u2
+        enum: track_mood
+        doc: |
+          The mood which rekordbox assigns the track as a whole during phrase analysis.
+      - size: 6
+      - id: end_beat
+        type: u2
+        doc: |
+          The beat number at which the last phrase ends. The track may
+          continue after the last phrase ends. If this is the case, it will
+          mostly be silence.
+      - size: 2
+      - id: raw_bank
+        type: u1
+        doc: |
+          Number identifying a stylistic bank which can be assigned to the track in rekordbox Lighting mode.
+      - size: 1
+      - id: entries
+        type: song_structure_entry
+        repeat: expr
+        repeat-expr: _parent.len_entries
+    instances:
+      bank:
+        value: raw_bank
+        enum: track_bank
+        doc: |
+          The stylistic bank which can be assigned to the track in rekordbox Lighting mode, if raw_bank has a legal value.
+        if: 'raw_bank < 9'
+
+
+  song_structure_entry:
+    doc: |
+      A song structure entry, represents a single phrase.
+    seq:
+      - id: index
+        type: u2
+        doc: |
+          The absolute number of the phrase, starting at one.
+      - id: beat
+        type: u2
+        doc: |
+          The beat number at which the phrase starts.
+      - id: kind
+        type:
+          switch-on: _parent.mood
+          cases:
+            'track_mood::high': phrase_high
+            'track_mood::mid': phrase_mid
+            'track_mood::low': phrase_low
+            _: phrase_mid  # We don't recognize this mood, so pick a generic interpretation.
+        doc: |
+          The kind of phrase as displayed in rekordbox.
+      - size: 1
+      - id: k1
+        type: u1
+        doc: One of three flags that identify phrase kind variants in high-mood tracks.
+      - size: 1
+      - id: k2
+        type: u1
+        doc: One of three flags that identify phrase kind variants in high-mood tracks.
+      - size: 1
+      - id: b
+        type: u1
+        doc: |
+          Flags how many more beat numbers are in a high-mood "Up 3" phrase.
+      - id: beat2
+        type: u2
+        doc: |
+          Extra beat number (falling within phrase) always present in high-mood "Up 3" phrases.
+      - id: beat3
+        type: u2
+        doc: |
+          Extra beat number (falling within phrase, larger than beat2)
+          present in high-mood "Up 3" phrases when b has value 1.
+      - id: beat4
+        type: u2
+        doc: |
+          Extra beat number (falling within phrase, larger than beat3)
+          present in high-mood "Up 3" phrases when b has value 1.
+      - size: 1
+      - id: k3
+        type: u1
+        doc: One of three flags that identify phrase kind variants in high-mood tracks.
+      - size: 1
+      - id: fill
+        type: u1
+        doc: |
+          If nonzero, fill-in is present at end of phrase.
+      - id: beat_fill
+        type: u2
+        doc: |
+          The beat number at which fill-in starts.
+    -webide-representation: '{kind.id}'
+
+  phrase_high:
+    seq:
+      - id: id
+        type: u2
+        enum: mood_high_phrase
+
+  phrase_mid:
+    seq:
+      - id: id
+        type: u2
+        enum: mood_mid_phrase
+
+  phrase_low:
+    seq:
+      - id: id
+        type: u2
+        enum: mood_low_phrase
+
+  unknown_tag: {}
+
+enums:
+  section_tags:
+    0x50434f42: cues                # PCOB
+    0x50434f32: cues_2              # PCO2 (seen in .EXT)
+    0x50505448: path                # PPTH
+    0x50564252: vbr                 # PVBR
+    0x5051545a: beat_grid           # PQTZ
+    0x50574156: wave_preview        # PWAV
+    0x50575632: wave_tiny           # PWV2
+    0x50575633: wave_scroll         # PWV3 (seen in .EXT)
+    0x50575634: wave_color_preview  # PWV4 (seen in .EXT)
+    0x50575635: wave_color_scroll   # PWV5 (seen in .EXT)
+    0x50535349: song_structure      # PSSI (seen in .EXT)
+    0x50575636: wave_3band_preview  # PWV6 (seen in .2EX)
+    0x50575637: wave_3band_scroll   # PWV7 (seen in .2EX)
+
+  cue_list_type:
+    0: memory_cues
+    1: hot_cues
+
+  cue_entry_type:
+    1: memory_cue
+    2: loop
+
+  cue_entry_status:
+    0: disabled
+    1: enabled
+    4: active_loop
+
+  track_mood:
+    1: high
+    2: mid
+    3: low
+
+  mood_low_phrase:
+    1: intro
+    2: verse_1
+    3: verse_1b  # Just displayed as "Verse 1" in rekordbox.
+    4: verse_1c  # Just displayed as "Verse 1" in rekordbox.
+    5: verse_2
+    6: verse_2b  # Just displayed as "Verse 2" in rekordbox.
+    7: verse_2c  # Just displayed as "Verse 2" in rekordbox.
+    8: bridge
+    9: chorus
+    10: outro
+
+  mood_mid_phrase:
+    1: intro
+    2: verse_1
+    3: verse_2
+    4: verse_3
+    5: verse_4
+    6: verse_5
+    7: verse_6
+    8: bridge
+    9: chorus
+    10: outro
+
+  mood_high_phrase:
+    1: intro
+    2: up
+    3: down
+    5: chorus
+    6: outro
+
+  track_bank:
+    0: default
+    1: cool
+    2: natural
+    3: hot
+    4: subtle
+    5: warm
+    6: vivid
+    7: club_1
+    8: club_2

maple-py/src/maple/artwork.py

@@ -0,0 +1,67 @@
+"""Extract embedded album art and write Pioneer-format JPEGs.
+
+Pioneer's CDJ/XDJ players expect two square JPEGs per artwork:
+  /PIONEER/Artwork/{bucket:05d}/a{id}.jpg     -> 80x80
+  /PIONEER/Artwork/{bucket:05d}/a{id}_m.jpg   -> 240x240
+where bucket = id // 20 + 1. The pdb's artwork_row.path points at the
+small (80x80) variant; the player auto-resolves the `_m` companion.
+"""
+import hashlib
+import io
+from pathlib import Path
+
+from mutagen.id3 import ID3, ID3NoHeaderError
+from PIL import Image
+
+SMALL = 80
+LARGE = 240
+
+
+def _read_apic(mp3: Path) -> bytes | None:
+    try:
+        tags = ID3(mp3)
+    except ID3NoHeaderError:
+        return None
+    frames = tags.getall("APIC")
+    if not frames:
+        return None
+    return frames[0].data
+
+
+def _bucket(artwork_id: int) -> str:
+    return f"{artwork_id // 20 + 1:05d}"
+
+
+def _resize_jpeg(data: bytes, size: int) -> bytes:
+    img = Image.open(io.BytesIO(data))
+    if img.mode != "RGB":
+        img = img.convert("RGB")
+    img = img.resize((size, size), Image.LANCZOS)
+    out = io.BytesIO()
+    img.save(out, format="JPEG", quality=85)
+    return out.getvalue()
+
+
+def extract_artwork(mp3: Path, usb_root: Path, next_id: int, cache: dict[str, int]) -> tuple[int, str | None]:
+    """Return (artwork_id, file_path_for_pdb). artwork_id is 0 if no art."""
+    data = _read_apic(mp3)
+    if data is None:
+        return 0, None
+    sha = hashlib.sha256(data).hexdigest()
+    cached = cache.get(sha)
+    if cached is not None:
+        return cached, None
+
+    aid = next_id
+    bucket = _bucket(aid)
+    rel_small = f"/PIONEER/Artwork/{bucket}/a{aid}.jpg"
+    rel_large = f"/PIONEER/Artwork/{bucket}/a{aid}_m.jpg"
+    abs_small = usb_root / rel_small.lstrip("/")
+    abs_large = usb_root / rel_large.lstrip("/")
+    abs_small.parent.mkdir(parents=True, exist_ok=True)
+    if not abs_small.exists():
+        abs_small.write_bytes(_resize_jpeg(data, SMALL))
+    if not abs_large.exists():
+        abs_large.write_bytes(_resize_jpeg(data, LARGE))
+    cache[sha] = aid
+    return aid, rel_small

maple-py/src/maple/cli.py

@@ -1,12 +1,16 @@
 """maple CLI: scan a directory of audio + m3u8 playlists and build a rekordbox USB."""
 import argparse
+import os
 import sys
+from concurrent.futures import ThreadPoolExecutor
 from datetime import date
 from pathlib import Path
 
-from maple.copy import stage_audio
+from maple.artwork import extract_artwork
+from maple.convert import convert_flac_to_mp3, verify_duration
+from maple.copy import stage_audio, staged_path
 from maple.meta import read_tags
-from maple.model import Album, Artist, Database, PlaylistEntry, PlaylistNode, Track
+from maple.model import Album, Artist, Artwork, Database, PlaylistEntry, PlaylistNode, Track
 from maple.pdb.writer import write_pdb
 from maple.scan import scan_audio, scan_playlists
 
@@ -18,10 +22,30 @@ def build(input_dir: Path, usb_root: Path) -> Database:
     artist_index: dict[str, int] = {}
     album_index: dict[tuple[int, str], int] = {}
     track_by_src: dict[Path, int] = {}
+    artwork_cache: dict[str, int] = {}
 
     audio_files = scan_audio(input_dir)
-    for src in audio_files:
+
-        meta = read_tags(src)
+    initial_meta = {p: read_tags(p) for p in audio_files}
+    flac_jobs: list[tuple[Path, Path]] = []
+    for original in audio_files:
+        if original.suffix.lower() == ".flac":
+            m = initial_meta[original]
+            dest_abs, _ = staged_path(usb_root, m.artist, m.album, original.with_suffix(".mp3").name)
+            flac_jobs.append((original, dest_abs))
+
+    def _job(pair: tuple[Path, Path]) -> tuple[Path, Path, bool]:
+        src, dst = pair
+        return src, dst, convert_flac_to_mp3(src, dst)
+
+    workers = max(1, os.cpu_count() or 1)
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        for src, dst, converted in ex.map(_job, flac_jobs):
+            print(f"{'convert' if converted else 'cached '} {src.relative_to(input_dir)}")
+            verify_duration(src, dst)
+
+    for original in audio_files:
+        meta = initial_meta[original]
         artist_id = artist_index.get(meta.artist)
         if artist_id is None:
             artist_id = len(db.artists) + 1
@@ -35,7 +59,17 @@ def build(input_dir: Path, usb_root: Path) -> Database:
             album_index[album_key] = album_id
             db.albums.append(Album(id=album_id, name=meta.album, artist_id=artist_id))
 
-        _, file_path = stage_audio(src, usb_root, meta.artist, meta.album)
+        if original.suffix.lower() == ".flac":
+            dest_abs, file_path = staged_path(usb_root, meta.artist, meta.album, original.with_suffix(".mp3").name)
+            meta = read_tags(dest_abs)
+        else:
+            dest_abs, file_path = stage_audio(original, usb_root, meta.artist, meta.album)
+
+        next_artwork_id = len(db.artwork) + 1
+        artwork_id, art_file_path = extract_artwork(dest_abs, usb_root, next_artwork_id, artwork_cache)
+        if art_file_path is not None:
+            db.artwork.append(Artwork(id=artwork_id, path=art_file_path))
+
         track_id = len(db.tracks) + 1
         db.tracks.append(Track(
             id=track_id,
@@ -52,9 +86,10 @@ def build(input_dir: Path, usb_root: Path) -> Database:
             track_number=meta.track_number,
             year=meta.year,
             date_added=today,
-            source_path=src,
+            artwork_id=artwork_id,
+            source_path=original,
         ))
-        track_by_src[src.resolve()] = track_id
+        track_by_src[original.resolve()] = track_id
 
     playlists = scan_playlists(input_dir)
     for sort_order, (name, paths) in enumerate(playlists.items(), start=1):
@@ -82,7 +117,7 @@ def build(input_dir: Path, usb_root: Path) -> Database:
 
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(prog="maple", description="Build a rekordbox USB from a folder of audio + m3u8 playlists.")
-    parser.add_argument("input_dir", type=Path)
+    parser.add_argument("input_dir", nargs="?", type=Path, default=Path("./data/input"))
     parser.add_argument("output_root", nargs="?", type=Path, default=Path("./data/usb"))
     args = parser.parse_args(argv)
 

maple-py/src/maple/convert.py

@@ -0,0 +1,82 @@
+"""FLAC -> MP3 V0 joint-stereo conversion via ffmpeg, with source-hash dedup.
+
+The SHA-256 of the source FLAC is stamped into a TXXX:MAPLE_SRC_SHA256 ID3
+frame on the produced MP3. Reruns skip conversion when the existing MP3
+carries the matching hash, so the cache is content-addressed against inputs.
+"""
+import hashlib
+import shutil
+import subprocess
+from pathlib import Path
+
+from mutagen import File as MutagenFile
+from mutagen.id3 import ID3, ID3NoHeaderError, TXXX
+
+HASH_FRAME_DESC = "MAPLE_SRC_SHA256"
+
+
+def file_sha256(path: Path) -> str:
+    with path.open("rb") as f:
+        return hashlib.file_digest(f, "sha256").hexdigest()
+
+
+def _read_source_hash(mp3: Path) -> str | None:
+    try:
+        tags = ID3(mp3)
+    except ID3NoHeaderError:
+        return None
+    for frame in tags.getall("TXXX"):
+        if frame.desc == HASH_FRAME_DESC and frame.text:
+            return str(frame.text[0])
+    return None
+
+
+def _write_source_hash(mp3: Path, sha: str) -> None:
+    try:
+        tags = ID3(mp3)
+    except ID3NoHeaderError:
+        tags = ID3()
+    tags.delall(f"TXXX:{HASH_FRAME_DESC}")
+    tags.add(TXXX(encoding=3, desc=HASH_FRAME_DESC, text=sha))
+    tags.save(mp3)
+
+
+def convert_flac_to_mp3(flac: Path, mp3: Path) -> bool:
+    """Convert flac to mp3 (V0, joint stereo). Returns True if conversion ran."""
+    src_hash = file_sha256(flac)
+    if mp3.exists() and _read_source_hash(mp3) == src_hash:
+        return False
+
+    if not shutil.which("ffmpeg"):
+        raise RuntimeError("ffmpeg not found on PATH")
+
+    mp3.parent.mkdir(parents=True, exist_ok=True)
+    subprocess.run(
+        [
+            "ffmpeg", "-nostdin", "-loglevel", "error", "-y",
+            "-i", str(flac),
+            "-codec:a", "libmp3lame",
+            "-q:a", "0",
+            "-joint_stereo", "1",
+            "-map_metadata", "0",
+            str(mp3),
+        ],
+        check=True,
+    )
+    _write_source_hash(mp3, src_hash)
+    return True
+
+
+def _duration(path: Path) -> float:
+    audio = MutagenFile(path)
+    info = getattr(audio, "info", None)
+    return float(getattr(info, "length", 0.0) or 0.0)
+
+
+def verify_duration(src: Path, dst: Path, tolerance: float = 0.1) -> None:
+    """Raise if dst's audio length differs from src's by more than `tolerance` seconds."""
+    a, b = _duration(src), _duration(dst)
+    if abs(a - b) > tolerance:
+        raise RuntimeError(
+            f"duration mismatch: {src.name} = {a:.3f}s vs {dst.name} = {b:.3f}s (Δ {abs(a-b):.3f}s)"
+        )

maple-py/src/maple/copy.py

@@ -1,24 +1,66 @@
-"""Copy audio files into the USB Contents layout."""
+"""Copy audio files into the USB Contents layout.
+
+Targets are assumed to be FAT32-formatted USBs, which constrain us to:
+  - per-component name length <= 255 LFN chars
+  - total path (from drive root) <= 260 chars
+  - no '\\ / : * ? " < > |' or control chars in any component
+We truncate per-component to a safe 64 chars (rekordbox firmware is
+fussier than raw FAT32) and refuse to emit anything that would still
+overflow the 260-char path budget.
+"""
 import re
 import shutil
 from pathlib import Path
 
 _BAD = re.compile(r'[\\/:*?"<>|\x00-\x1f]')
 
+MAX_COMPONENT = 48
+MAX_PATH = 260
+# Pioneer firmware appears to only handle the DeviceSQL short-ASCII form
+# for `file_path` in the pdb (length byte < 127). The verified reference
+# export uses short-form for every track. Keep the on-pdb path strictly
+# inside that budget so every track stays loadable.
+MAX_PDB_PATH = 126
 
-def sanitise(component: str) -> str:
+
+def sanitise(component: str, max_len: int = MAX_COMPONENT) -> str:
     s = _BAD.sub("_", component).strip(" .")
+    if len(s) > max_len:
+        s = s[:max_len].rstrip(" .")
     return s or "_"
 
 
-def stage_audio(src: Path, usb_root: Path, artist: str, album: str) -> tuple[Path, str]:
+def _truncate_filename(name: str, max_len: int) -> str:
-    artist_dir = sanitise(artist)
+    """Truncate while preserving the extension."""
-    album_dir = sanitise(album)
+    cleaned = _BAD.sub("_", name).strip(" .")
-    filename = sanitise(src.name)
+    if len(cleaned) <= max_len:
-    dest_rel = Path("Contents") / artist_dir / album_dir / filename
+        return cleaned or "_"
+    stem, dot, ext = cleaned.rpartition(".")
+    if dot and len(ext) < max_len - 1:
+        keep = max_len - len(ext) - 1
+        return (stem[:keep].rstrip(" .") + "." + ext) or "_"
+    return cleaned[:max_len].rstrip(" .") or "_"
+
+
+def staged_path(usb_root: Path, artist: str, album: str, filename: str) -> tuple[Path, str]:
+    artist_s = sanitise(artist)
+    album_s = sanitise(album)
+    prefix = f"/Contents/{artist_s}/{album_s}/"
+    budget = min(MAX_COMPONENT, MAX_PDB_PATH - len(prefix))
+    if budget < 8:
+        raise ValueError(f"no room for filename within MAX_PDB_PATH={MAX_PDB_PATH} (prefix uses {len(prefix)}): {prefix}")
+    name_s = _truncate_filename(filename, budget)
+    dest_rel = Path("Contents") / artist_s / album_s / name_s
     dest_abs = usb_root / dest_rel
+    file_path = "/" + dest_rel.as_posix()
+    if len(file_path) > MAX_PDB_PATH:
+        raise ValueError(f"pdb path budget exceeded ({len(file_path)} > {MAX_PDB_PATH}): {file_path}")
+    return dest_abs, file_path
+
+
+def stage_audio(src: Path, usb_root: Path, artist: str, album: str) -> tuple[Path, str]:
+    dest_abs, file_path = staged_path(usb_root, artist, album, src.name)
     dest_abs.parent.mkdir(parents=True, exist_ok=True)
     if not dest_abs.exists():
         shutil.copy2(src, dest_abs)
-    file_path = "/" + dest_rel.as_posix()
     return dest_abs, file_path

maple-py/src/maple/model.py

@@ -39,9 +39,57 @@ class Track:
     mix_name: str = ""
     analyze_path: str = ""
     analyze_date: str = ""
+    artwork_id: int = 0
     source_path: Path | None = None
 
 
+@dataclass
+class Artwork:
+    id: int
+    path: str
+
+
+@dataclass
+class Column:
+    id: int
+    code: int
+    name: str
+
+
+# Reverse-engineered from a verified rekordbox export. Each entry is
+# (column_id, internal column code, display name). Code values are
+# non-contiguous because rekordbox skips reserved slots.
+DEFAULT_COLUMNS: list[Column] = [
+    Column(1, 0x80, "GENRE"),
+    Column(2, 0x81, "ARTIST"),
+    Column(3, 0x82, "ALBUM"),
+    Column(4, 0x83, "TRACK"),
+    Column(5, 0x85, "BPM"),
+    Column(6, 0x86, "RATING"),
+    Column(7, 0x87, "YEAR"),
+    Column(8, 0x88, "REMIXER"),
+    Column(9, 0x89, "LABEL"),
+    Column(10, 0x8A, "ORIGINAL ARTIST"),
+    Column(11, 0x8B, "KEY"),
+    Column(12, 0x8D, "CUE"),
+    Column(13, 0x8E, "COLOR"),
+    Column(14, 0x92, "TIME"),
+    Column(15, 0x93, "BITRATE"),
+    Column(16, 0x94, "FILE NAME"),
+    Column(17, 0x84, "PLAYLIST"),
+    Column(18, 0x98, "HOT CUE BANK"),
+    Column(19, 0x95, "HISTORY"),
+    Column(20, 0x91, "SEARCH"),
+    Column(21, 0x96, "COMMENTS"),
+    Column(22, 0x8C, "DATE ADDED"),
+    Column(23, 0x97, "DJ PLAY COUNT"),
+    Column(24, 0x90, "FOLDER"),
+    Column(25, 0xA1, "DEFAULT"),
+    Column(26, 0xA2, "ALPHABET"),
+    Column(27, 0xAA, "MATCHING"),
+]
+
+
 @dataclass
 class PlaylistNode:
     id: int
@@ -65,3 +113,4 @@ class Database:
     albums: list[Album] = field(default_factory=list)
     playlist_nodes: list[PlaylistNode] = field(default_factory=list)
     playlist_entries: list[PlaylistEntry] = field(default_factory=list)
+    artwork: list[Artwork] = field(default_factory=list)

maple-py/src/maple/pdb/pager.py

@@ -4,13 +4,42 @@ from struct import pack
 PAGE_SIZE = 4096
 HEADER_SIZE = 40
 ROW_GROUP_SIZE = 36
-MAX_ROWS_PER_PAGE = 16  # one row group per page, matching rekordcrate
+ROWS_PER_GROUP = 16
 
 
 def _align4(offset: int) -> int:
     return (offset + 3) & ~3
 
 
+def _num_row_groups(n: int) -> int:
+    return (n + ROWS_PER_GROUP - 1) // ROWS_PER_GROUP
+
+
+def build_strange_page(page_index: int, page_type: int, next_page: int) -> bytes:
+    """Empty placeholder page that the reference exports use as the first
+    page of every table chain. Reproduces the observed header flags and
+    marker bytes; the heap is left zero-filled (rekordbox fills it with a
+    `f8 ff ff 1f` pattern, but players don't care because num_rows=0).
+    """
+    header = b""
+    header += pack("<I", 0)                       # magic
+    header += pack("<I", page_index)
+    header += pack("<I", page_type)
+    header += pack("<I", next_page)
+    header += pack("<I", 1)                       # sequence
+    header += pack("<I", 0)                       # reserved
+    # packed: num_row_offsets=0, num_rows=0, page_flags=0x64
+    header += bytes([0, 0, 0, 0x64])
+    header += pack("<H", 0x1fff)                  # free_size (strange marker)
+    header += pack("<H", 0x1fff)                  # used_size (strange marker)
+    header += pack("<H", 0x1fff)                  # transaction_row_count = failed
+    header += pack("<H", 0x1fff)                  # transaction_row_index = failed
+    header += pack("<H", 0x03ec)                  # "1004 for strange blocks"
+    header += pack("<H", 0)                       # unknown
+    assert len(header) == HEADER_SIZE
+    return header + b"\x00" * (PAGE_SIZE - HEADER_SIZE)
+
+
 class PageBuilder:
     def __init__(self, page_index: int, page_type: int):
         self.page_index = page_index
@@ -21,12 +50,10 @@ class PageBuilder:
         self.heap_cursor = 0  # position relative to heap_pos
 
     def try_add(self, row: bytes) -> bool:
-        if len(self.row_bytes) >= MAX_ROWS_PER_PAGE:
-            return False
         aligned = _align4(self.heap_cursor)
         end = aligned + len(row)
-        heap_capacity = PAGE_SIZE - HEADER_SIZE - ROW_GROUP_SIZE
+        footer_after = _num_row_groups(len(self.row_bytes) + 1) * ROW_GROUP_SIZE
-        if end > heap_capacity:
+        if end + HEADER_SIZE + footer_after > PAGE_SIZE:
             return False
         self.row_offsets.append(aligned)
         self.row_bytes.append(row)
@@ -38,17 +65,17 @@ class PageBuilder:
 
     def finalize(self) -> bytes:
         n = len(self.row_bytes)
-        if n == 0:
+        num_rgs = _num_row_groups(n)
-            used_size = 0
+        footer_size = num_rgs * ROW_GROUP_SIZE
-            heap_padding = PAGE_SIZE - HEADER_SIZE
+        heap_padding = PAGE_SIZE - HEADER_SIZE - footer_size
-        else:
+        used_size = self.heap_cursor
-            used_size = self.heap_cursor
-            heap_padding = PAGE_SIZE - HEADER_SIZE - ROW_GROUP_SIZE
         free_size = heap_padding - used_size
         if free_size < 0:
             raise ValueError("page overflow")
 
-        page_flags = 0x00 if n > 0 else 0x44
+        # Reference exports use 0x24 for data pages with rows. Empty data
+        # pages (post-strange) keep 0x44 to mark them as having no rows.
+        page_flags = 0x24 if n > 0 else 0x44
 
         header = b""
         header += pack("<I", 0)                       # magic
@@ -57,32 +84,38 @@ class PageBuilder:
         header += pack("<I", self.next_page)
         header += pack("<I", 0)                       # unknown1
         header += pack("<I", 0)                       # unknown2
-        header += pack("<BBBB", n & 0xFF, 0, 0, page_flags)
+        # 24 bits little-endian: low 13 = num_row_offsets, next 11 = num_rows,
+        # then u1 page_flags. Kaitai reads these as read_bits_int_le.
+        packed = ((n & 0x7FF) << 13) | (n & 0x1FFF)
+        header += bytes([packed & 0xFF, (packed >> 8) & 0xFF, (packed >> 16) & 0xFF, page_flags])
         header += pack("<HH", free_size, used_size)
-        header += pack("<H", 0)                       # unknown5
+        header += pack("<H", 0)                       # transaction_row_count
-        header += pack("<H", n)                       # num_rows_large
+        header += pack("<H", 0)                       # transaction_row_index
         header += pack("<H", 0)                       # unknown6
         header += pack("<H", 0)                       # unknown7
         assert len(header) == HEADER_SIZE
 
         heap = bytearray(heap_padding)
-        if n > 0:
+        for ofs, row in zip(self.row_offsets, self.row_bytes):
-            for ofs, row in zip(self.row_offsets, self.row_bytes):
+            heap[ofs:ofs + len(row)] = row
-                heap[ofs:ofs + len(row)] = row
 
-        if n == 0:
+        # Footers go after the heap in the file. Group 0 (rows 0-15) sits closest
-            return header + bytes(heap)
+        # to the end of the page; group num_rgs-1 sits closest to the heap. So
+        # emit footers in reverse group order.
+        footer = b""
+        for g in range(num_rgs - 1, -1, -1):
+            start = g * ROWS_PER_GROUP
+            grp_offsets = self.row_offsets[start:start + ROWS_PER_GROUP]
+            grp_count = len(grp_offsets)
+            padded = grp_offsets + [0] * (ROWS_PER_GROUP - grp_count)
+            presence = (1 << grp_count) - 1
+            rg = b""
+            for ofs in reversed(padded):
+                rg += pack("<H", ofs)
+            rg += pack("<H", presence)
+            rg += pack("<H", 16)  # unknown (rekordcrate default)
+            assert len(rg) == ROW_GROUP_SIZE
+            footer += rg
+        assert len(footer) == footer_size
 
-        # Row group footer (36 bytes), placed at end of page
+        return header + bytes(heap) + footer
-        offsets_padded = self.row_offsets + [0] * (16 - n)
-        presence = 0
-        for i in range(n):
-            presence |= 1 << i
-        rg = b""
-        for ofs in reversed(offsets_padded):
-            rg += pack("<H", ofs)
-        rg += pack("<H", presence)
-        rg += pack("<H", 16)  # unknown (rekordcrate default)
-        assert len(rg) == ROW_GROUP_SIZE
-
-        return header + bytes(heap) + rg

maple-py/src/maple/pdb/rows.py

@@ -1,7 +1,7 @@
 """Row serialisers for the rekordbox export.pdb format."""
 from struct import pack
 
-from maple.model import Album, Artist, PlaylistEntry, PlaylistNode, Track
+from maple.model import Album, Artist, Artwork, Column, PlaylistEntry, PlaylistNode, Track
 from maple.pdb.strings import encode_devicesql
 
 TRACK_FIXED_PREFIX = 0x5E
@@ -58,7 +58,7 @@ def serialize_track(t: Track) -> bytes:
     prefix += pack("<I", 0)                     # unknown
     prefix += pack("<H", 19048)                 # unknown ("always 19048?")
     prefix += pack("<H", 30967)                 # unknown ("always 30967?")
-    prefix += pack("<I", 0)                     # artwork_id
+    prefix += pack("<I", t.artwork_id)          # artwork_id
     prefix += pack("<I", 0)                     # key_id
     prefix += pack("<I", 0)                     # original_artist_id
     prefix += pack("<I", 0)                     # label_id
@@ -145,3 +145,19 @@ def serialize_playlist_tree(n: PlaylistNode) -> bytes:
 
 def serialize_playlist_entry(e: PlaylistEntry) -> bytes:
     return pack("<III", e.entry_index, e.track_id, e.playlist_id)
+
+
+def serialize_artwork(a: Artwork) -> bytes:
+    row = pack("<I", a.id) + encode_devicesql(a.path)
+    return _pad2(row)
+
+
+def serialize_column(c: Column) -> bytes:
+    """A column row is: u2 id, u2 code, then a 'long' devicesql string whose
+    UTF-16 body is wrapped by the sentinel codepoints U+FFFA / U+FFFB.
+    """
+    body = b"\xfa\xff" + c.name.encode("utf-16-le") + b"\xfb\xff"
+    total_length = len(body) + 4  # 4 = 0x90 marker + u2 length + u1 pad
+    name_str = b"\x90" + pack("<H", total_length) + b"\x00" + body
+    row = pack("<HH", c.id, c.code) + name_str
+    return _pad2(row)

maple-py/src/maple/pdb/strings.py

@@ -1,4 +1,15 @@
-"""DeviceSQL variable-length string encoding."""
+"""DeviceSQL variable-length string encoding.
+
+Three on-disk forms (see rekordbox-pdb.ksy `device_sql_string`):
+  - odd length_and_kind byte -> short ASCII (< 127 chars)
+  - 0x40                     -> long ASCII  (any length, ASCII bytes)
+  - 0x90                     -> long UTF-16LE (for non-ASCII content)
+
+We pick the cheapest form that fits the content. Long-but-ASCII paths
+must use 0x40, NOT 0x90 -- some player firmwares decode 0x90 strings
+as UTF-16 and then fail the case-sensitive filesystem lookup, so any
+track whose `file_path` lands in the wrong form silently goes missing.
+"""
 
 EMPTY = b"\x03"
 
@@ -10,10 +21,15 @@ def encode_devicesql(s: str) -> bytes:
         ascii_bytes = s.encode("ascii")
     except UnicodeEncodeError:
         ascii_bytes = None
-    if ascii_bytes is not None and len(ascii_bytes) < 127:
+
-        length = len(ascii_bytes) + 1
+    if ascii_bytes is not None:
-        length_and_kind = ((length << 1) | 1) & 0xFF
+        if len(ascii_bytes) < 127:
-        return bytes([length_and_kind]) + ascii_bytes
+            length = len(ascii_bytes) + 1
+            length_and_kind = ((length << 1) | 1) & 0xFF
+            return bytes([length_and_kind]) + ascii_bytes
+        total_length = len(ascii_bytes) + 4  # marker + u2 length + u1 pad
+        return bytes([0x40]) + total_length.to_bytes(2, "little") + b"\x00" + ascii_bytes
+
     payload = s.encode("utf-16-le")
     total_length = len(payload) + 4
     return bytes([0x90]) + total_length.to_bytes(2, "little") + b"\x00" + payload

maple-py/src/maple/pdb/writer.py

@@ -2,56 +2,86 @@
 from pathlib import Path
 from struct import pack
 
-from maple.model import Database
+from maple.model import DEFAULT_COLUMNS, Database
-from maple.pdb.pager import PAGE_SIZE, PageBuilder
+from maple.pdb.pager import PAGE_SIZE, PageBuilder, build_strange_page
 from maple.pdb.rows import (
     serialize_album,
     serialize_artist,
+    serialize_artwork,
+    serialize_column,
     serialize_playlist_entry,
     serialize_playlist_tree,
     serialize_track,
 )
 
 PAGE_TYPE_TRACKS = 0
+PAGE_TYPE_GENRES = 1
 PAGE_TYPE_ARTISTS = 2
 PAGE_TYPE_ALBUMS = 3
+PAGE_TYPE_LABELS = 4
+PAGE_TYPE_KEYS = 5
+PAGE_TYPE_COLORS = 6
 PAGE_TYPE_PLAYLIST_TREE = 7
 PAGE_TYPE_PLAYLIST_ENTRIES = 8
+PAGE_TYPE_UNKNOWN_9 = 9
+PAGE_TYPE_UNKNOWN_10 = 10
+PAGE_TYPE_HISTORY_PLAYLISTS = 11
+PAGE_TYPE_HISTORY_ENTRIES = 12
+PAGE_TYPE_ARTWORK = 13
+PAGE_TYPE_UNKNOWN_14 = 14
+PAGE_TYPE_UNKNOWN_15 = 15
+PAGE_TYPE_COLUMNS = 16
+PAGE_TYPE_UNKNOWN_17 = 17
+PAGE_TYPE_UNKNOWN_18 = 18
+PAGE_TYPE_HISTORY = 19
+
+ALL_PAGE_TYPES = list(range(20))
 
 FILE_HEADER_SIZE = 28  # 0,page_size,num_tables,next_unused,unknown,sequence,gap = 7*u4
 TABLE_ENTRY_SIZE = 16  # page_type,empty_candidate,first_page,last_page = 4*u4
 
 
 def _build_chain(page_type: int, row_iter, start_index: int) -> tuple[list[bytes], int, int]:
-    """Return (page_bytes_list, first_page, last_page)."""
+    """Return (page_bytes_list, first_page, last_page).
-    pages: list[PageBuilder] = []
-    page_idx = start_index
-    current = PageBuilder(page_idx, page_type)
 
-    for row in row_iter:
+    Every table chain begins with a "strange" placeholder page (flags=0x64)
+    mirroring rekordbox's own exports. Data pages with rows follow.
+    Tables with no rows emit only the strange page.
+    """
+    rows = list(row_iter)
+    out: list[bytes] = []
+
+    # Reserve start_index for the strange page; data pages follow.
+    if not rows:
+        # Strange page only: it points to itself (end of chain).
+        out.append(build_strange_page(start_index, page_type, next_page=start_index))
+        return out, start_index, start_index
+
+    # Build data pages starting at start_index + 1.
+    data_pages: list[PageBuilder] = []
+    page_idx = start_index + 1
+    current = PageBuilder(page_idx, page_type)
+    for row in rows:
         if not current.try_add(row):
-            pages.append(current)
+            data_pages.append(current)
             page_idx += 1
             current = PageBuilder(page_idx, page_type)
             if not current.try_add(row):
                 raise ValueError(f"row of size {len(row)} too large for an empty page")
+    data_pages.append(current)
 
-    pages.append(current)
+    # Strange page links to the first data page.
+    out.append(build_strange_page(start_index, page_type, next_page=start_index + 1))
 
-    # If the chain has no rows at all, still emit one empty page
+    # Patch each data page's next_page; last points to itself.
-    first_page = start_index
+    for i, p in enumerate(data_pages):
-    last_page = page_idx
+        if i == len(data_pages) - 1:
-
-    # Patch next_page pointers: each non-last page points to the next; last points to itself
-    out: list[bytes] = []
-    for i, p in enumerate(pages):
-        if i == len(pages) - 1:
             p.next_page = p.page_index
         else:
             p.next_page = p.page_index + 1
         out.append(p.finalize())
 
-    return out, first_page, last_page
+    return out, start_index, page_idx
 
 
 def write_pdb(db: Database, path: str | Path) -> None:
@@ -64,14 +94,24 @@ def write_pdb(db: Database, path: str | Path) -> None:
     serialized_albums = [serialize_album(a) for a in db.albums]
     serialized_pl_tree = [serialize_playlist_tree(n) for n in db.playlist_nodes]
     serialized_pl_entries = [serialize_playlist_entry(e) for e in db.playlist_entries]
+    serialized_artwork = [serialize_artwork(a) for a in db.artwork]
+    serialized_columns = [serialize_column(c) for c in DEFAULT_COLUMNS]
 
-    for ptype, rows in [
+    rows_by_type: dict[int, list[bytes]] = {
-        (PAGE_TYPE_TRACKS, serialized_tracks),
+        PAGE_TYPE_TRACKS: serialized_tracks,
-        (PAGE_TYPE_ARTISTS, serialized_artists),
+        PAGE_TYPE_ARTISTS: serialized_artists,
-        (PAGE_TYPE_ALBUMS, serialized_albums),
+        PAGE_TYPE_ALBUMS: serialized_albums,
-        (PAGE_TYPE_PLAYLIST_TREE, serialized_pl_tree),
+        PAGE_TYPE_PLAYLIST_TREE: serialized_pl_tree,
-        (PAGE_TYPE_PLAYLIST_ENTRIES, serialized_pl_entries),
+        PAGE_TYPE_PLAYLIST_ENTRIES: serialized_pl_entries,
-    ]:
+        PAGE_TYPE_ARTWORK: serialized_artwork,
+        PAGE_TYPE_COLUMNS: serialized_columns,
+    }
+    # Emit all 20 page-type chains; tables we don't populate get a single empty page.
+    # CDJ/XDJ firmware looks up tables by index, so emitting the full set keeps
+    # features that probe optional tables (artwork, keys, colors, history) from
+    # failing the lookup.
+    for ptype in ALL_PAGE_TYPES:
+        rows = rows_by_type.get(ptype, [])
         pages, first, last = _build_chain(ptype, rows, next_page_idx)
         chains.append((ptype, pages, first, last))
         next_page_idx = last + 1

maple-py/uv.lock

@@ -19,12 +19,23 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484 },
 ]
 
+[[package]]
+name = "kaitaistruct"
+version = "0.11"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/27/b8/ca7319556912f68832daa4b81425314857ec08dfccd8dbc8c0f65c992108/kaitaistruct-0.11.tar.gz", hash = "sha256:053ee764288e78b8e53acf748e9733268acbd579b8d82a427b1805453625d74b", size = 11519 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/4a/4a/cf14bf3b1f5ffb13c69cf5f0ea78031247790558ee88984a8bdd22fae60d/kaitaistruct-0.11-py2.py3-none-any.whl", hash = "sha256:5c6ce79177b4e193a577ecd359e26516d1d6d000a0bffd6e1010f2a46a62a561", size = 11372 },
+]
+
 [[package]]
 name = "maple"
 version = "0.1.0"
 source = { editable = "." }
 dependencies = [
+    { name = "kaitaistruct" },
     { name = "mutagen" },
+    { name = "pillow" },
 ]
 
 [package.dev-dependencies]
@@ -33,7 +44,11 @@ dev = [
 ]
 
 [package.metadata]
-requires-dist = [{ name = "mutagen", specifier = ">=1.47.0" }]
+requires-dist = [
+    { name = "kaitaistruct", specifier = ">=0.11" },
+    { name = "mutagen", specifier = ">=1.47.0" },
+    { name = "pillow", specifier = ">=12.2.0" },
+]
 
 [package.metadata.requires-dev]
 dev = [{ name = "pytest", specifier = ">=9.0.3" }]
@@ -56,6 +71,39 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195 },
 ]
 
+[[package]]
+name = "pillow"
+version = "12.2.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/bf/98/4595daa2365416a86cb0d495248a393dfc84e96d62ad080c8546256cb9c0/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8", size = 4100848 },
+    { url = "https://files.pythonhosted.org/packages/0b/79/40184d464cf89f6663e18dfcf7ca21aae2491fff1a16127681bf1fa9b8cf/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b", size = 4176515 },
+    { url = "https://files.pythonhosted.org/packages/b0/63/703f86fd4c422a9cf722833670f4f71418fb116b2853ff7da722ea43f184/pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295", size = 3640159 },
+    { url = "https://files.pythonhosted.org/packages/71/e0/fb22f797187d0be2270f83500aab851536101b254bfa1eae10795709d283/pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed", size = 5312185 },
+    { url = "https://files.pythonhosted.org/packages/ba/8c/1a9e46228571de18f8e28f16fabdfc20212a5d019f3e3303452b3f0a580d/pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae", size = 4695386 },
+    { url = "https://files.pythonhosted.org/packages/70/62/98f6b7f0c88b9addd0e87c217ded307b36be024d4ff8869a812b241d1345/pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601", size = 6280384 },
+    { url = "https://files.pythonhosted.org/packages/5e/03/688747d2e91cfbe0e64f316cd2e8005698f76ada3130d0194664174fa5de/pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be", size = 8091599 },
+    { url = "https://files.pythonhosted.org/packages/f6/35/577e22b936fcdd66537329b33af0b4ccfefaeabd8aec04b266528cddb33c/pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f", size = 6396021 },
+    { url = "https://files.pythonhosted.org/packages/11/8d/d2532ad2a603ca2b93ad9f5135732124e57811d0168155852f37fbce2458/pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286", size = 7083360 },
+    { url = "https://files.pythonhosted.org/packages/5e/26/d325f9f56c7e039034897e7380e9cc202b1e368bfd04d4cbe6a441f02885/pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50", size = 6507628 },
+    { url = "https://files.pythonhosted.org/packages/5f/f7/769d5632ffb0988f1c5e7660b3e731e30f7f8ec4318e94d0a5d674eb65a4/pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104", size = 7209321 },
+    { url = "https://files.pythonhosted.org/packages/6a/7a/c253e3c645cd47f1aceea6a8bacdba9991bf45bb7dfe927f7c893e89c93c/pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7", size = 6479723 },
+    { url = "https://files.pythonhosted.org/packages/cd/8b/601e6566b957ca50e28725cb6c355c59c2c8609751efbecd980db44e0349/pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150", size = 7217400 },
+    { url = "https://files.pythonhosted.org/packages/d6/94/220e46c73065c3e2951bb91c11a1fb636c8c9ad427ac3ce7d7f3359b9b2f/pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1", size = 2554835 },
+    { url = "https://files.pythonhosted.org/packages/b6/ab/1b426a3974cb0e7da5c29ccff4807871d48110933a57207b5a676cccc155/pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463", size = 5314225 },
+    { url = "https://files.pythonhosted.org/packages/19/1e/dce46f371be2438eecfee2a1960ee2a243bbe5e961890146d2dee1ff0f12/pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3", size = 4698541 },
+    { url = "https://files.pythonhosted.org/packages/55/c3/7fbecf70adb3a0c33b77a300dc52e424dc22ad8cdc06557a2e49523b703d/pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166", size = 6322251 },
+    { url = "https://files.pythonhosted.org/packages/1c/3c/7fbc17cfb7e4fe0ef1642e0abc17fc6c94c9f7a16be41498e12e2ba60408/pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe", size = 8127807 },
+    { url = "https://files.pythonhosted.org/packages/ff/c3/a8ae14d6defd2e448493ff512fae903b1e9bd40b72efb6ec55ce0048c8ce/pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd", size = 6433935 },
+    { url = "https://files.pythonhosted.org/packages/6e/32/2880fb3a074847ac159d8f902cb43278a61e85f681661e7419e6596803ed/pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e", size = 7116720 },
+    { url = "https://files.pythonhosted.org/packages/46/87/495cc9c30e0129501643f24d320076f4cc54f718341df18cc70ec94c44e1/pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06", size = 6540498 },
+    { url = "https://files.pythonhosted.org/packages/18/53/773f5edca692009d883a72211b60fdaf8871cbef075eaa9d577f0a2f989e/pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43", size = 7239413 },
+    { url = "https://files.pythonhosted.org/packages/c9/e4/4b64a97d71b2a83158134abbb2f5bd3f8a2ea691361282f010998f339ec7/pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354", size = 6482084 },
+    { url = "https://files.pythonhosted.org/packages/ba/13/306d275efd3a3453f72114b7431c877d10b1154014c1ebbedd067770d629/pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1", size = 7225152 },
+    { url = "https://files.pythonhosted.org/packages/ff/6e/cf826fae916b8658848d7b9f38d88da6396895c676e8086fc0988073aaf8/pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb", size = 2556579 },
+]
+
 [[package]]
 name = "pluggy"
 version = "1.6.0"