From 6a6553077be14fb1592671f65862c990de05310b Mon Sep 17 00:00:00 2001 From: Nathan Rashleigh Date: Tue, 26 May 2026 18:14:46 +1000 Subject: [PATCH] idk --- etc/rekordbox-pdb.ksy | 99 ++- justfile | 9 +- maple-py/src/kaitai/rekordbox_anlz.py | 897 ++++++++++++++++++++++++++ maple-py/src/kaitai/rekordbox_pdb.py | 72 ++- maple-py/src/maple/anlz.py | 41 ++ maple-py/src/maple/cli.py | 33 +- maple-py/src/maple/meta.py | 21 + maple-py/src/maple/model.py | 50 ++ maple-py/src/maple/pdb/pager.py | 13 +- maple-py/src/maple/pdb/rows.py | 59 +- maple-py/src/maple/pdb/writer.py | 53 +- 11 files changed, 1315 insertions(+), 32 deletions(-) create mode 100644 maple-py/src/kaitai/rekordbox_anlz.py create mode 100644 maple-py/src/maple/anlz.py diff --git a/etc/rekordbox-pdb.ksy b/etc/rekordbox-pdb.ksy index 152bdce..c54d22d 100644 --- a/etc/rekordbox-pdb.ksy +++ b/etc/rekordbox-pdb.ksy @@ -355,6 +355,8 @@ types: 'page_type::history_playlists': history_playlist_row 'page_type::history_entries': history_entry_row 'page_type::tracks': track_row + 'page_type::browse_filter_index': browse_filter_index_row + 'page_type::browse_menu': browse_menu_row doc: | The actual content of the row, as long as it is present. -webide-parse-mode: eager @@ -605,6 +607,78 @@ types: doc: | The variable-length string naming the playlist. + browse_filter_index_row: + doc: | + A row in the `browse_filter_index` (was `unknown_17`) table. Lists the columns + that appear in the player's "Filter by ->" sidebar, in display order. The + reverse-engineered semantics: each row enumerates one filterable column. The + first group (`d == 0`) has 12 rows that look like type/icon descriptors; the + second group (`d == 1..10`) defines the navigation entries with `d` as the + sort position. The 22 column ids that appear here are exactly the + "filterable" subset of the 27 columns -- CUE, COMMENTS, DJ PLAY COUNT, + DEFAULT, ALPHABET do not appear. Verified byte-for-byte identical across + multiple working rekordbox exports. + seq: + - id: column_id + type: u2 + doc: | + The column this row references; corresponds to `id` in the `columns` table. + - id: secondary_column + type: u2 + doc: | + A second column id whose role is not fully nailed down. For rows in the + `d == 0` group it usually equals `column_id + 1`; for the navigation + group it varies. Likely a secondary sort key or an internal ID-lookup + target rather than something rendered to the user (the KEY filter does + not show CUEs even though `secondary_column == CUE` here). + - id: filter_type + type: u2 + doc: | + Filter kind code. Observed values: 0x163 (=355) for textual filters, + 0x104-0x106 (=260-262) for numeric/enum filters, 0x63 (=99) for + non-leaf navigation entries, 0x263 (=611) for the MATCHING entry, + 0x01-0x03 for the ARTIST/ALBUM/TRACK leaf entries. + - id: sort_index + type: u2 + doc: | + Zero for the first group, otherwise 1..10 giving the display order of + this row in the top-level filter list (1 = first, 10 = last). + + browse_menu_row: + doc: | + A row in the `browse_menu` (was `unknown_18`) table. Together with + `browse_filter_index`, this drives the player's top-level browse navigation. + Semantic model: `display_column` is the column shown in the menu; + `sort_column` is the column used to sort/key it. When the two are equal the + entry opens a drilldown of that column's distinct values (e.g. + `(ARTIST, ARTIST)` -> list of artists). When they differ the entry opens a + flat list of `display_column` rows sorted by `sort_column` (e.g. + `(BPM, TRACK)` -> tracks sorted by BPM). Verified content is identical + across all observed working exports. + seq: + - id: sort_column + type: u2 + doc: | + The column used to order the list shown by this menu entry. + - id: display_column + type: u2 + doc: | + The column whose rows the entry lists. If `display_column == sort_column`, + you get a drilldown of that column's distinct values; otherwise a flat + list ordered by `sort_column`. + - id: flags + type: u2 + doc: | + Two packed bytes. Low byte is non-zero only for the "tracks sorted by + BPM" row (=2, possibly a rendering hint). High byte (0x01..0x07) gives + the display order of the top-level entries (1=DEFAULT, 2=ALPHABET, + 3=ARTIST, 4=ALBUM, 5=tracks-by-BPM, 6=BPM-by-rating, 7=cues-by-key). + A value of 1 marks the row as a secondary sort pair rather than a + top-level menu item. + - id: reserved + type: u2 + doc: Always zero in observed exports. + history_entry_row: doc: | A row that associates a track with a position in a history playlist. @@ -1138,9 +1212,30 @@ enums: doc: | TODO figure out and explain 17: - id: unknown_17 + id: browse_filter_index + doc: | + Drives the player's "Filter by ->" sidebar. Each row is four little-endian u2s + `(a, b, c, d)` -- see `browse_filter_index_row` -- where `a` is the column the + menu item shows, `d` is its sort order in the filter list, and `c` is a type + code (0x163/0x104/0x105/0x106 for the various filter kinds). Exactly the + columns not in this table are the "non-filterable" ones (CUE, COMMENTS, + DJ PLAY COUNT, DEFAULT, ALPHABET); 22 of the 27 column ids appear here. + Verified content is identical across multiple working exports, so this is + effectively a fixed table rather than per-library content. Was `unknown_17`. 18: - id: unknown_18 + id: browse_menu + doc: | + Drives the player's top-level browse menu. Each row is four little-endian u2s + `(a, b, c, d)` -- see `browse_menu_row` -- where `b` is the column the menu + entry *displays* and `a` is the column it *sorts by*. When `a == b` the entry + opens a drilldown of that column's distinct values (e.g. ARTIST -> list of + artists). When `a != b` the entry opens a flat list of `b` rows sorted by + `a` (e.g. (BPM, TRACK) -> tracks sorted by BPM). Rows split into two groups: + rows where `c == 1` are secondary sort pairs (10 of them); rows where `c` + is `(0x100 * n)` define the top-level menu items with `n` as the display + position (DEFAULT, ALPHABET, ARTIST, ALBUM, "tracks sorted by BPM", + "BPM by rating", "cues by key"). Verified content is identical across all + observed working exports. Was `unknown_18`. 19: id: history doc: | diff --git a/justfile b/justfile index fd38183..b83e1ac 100644 --- a/justfile +++ b/justfile @@ -33,15 +33,18 @@ sync args: usb dst="/Volumes/MAPLE": COPYFILE_DISABLE=1 rsync -rtvh --delete --size-only --no-perms --no-owner --no-group --modify-window=2 maple-py/data/usb/Contents/ {{dst}}/Contents/ - COPYFILE_DISABLE=1 rsync -rtvh --delete --size-only --no-perms --no-owner --no-group --modify-window=2 maple-py/data/usb/PIONEER/ {{dst}}/PIONEER/ - # find {{dst}} -name '._*' -delete + COPYFILE_DISABLE=1 rsync -rtvh --delete --size-only --no-perms --no-owner --no-group --modify-window=2 --exclude='USBANLZ/' maple-py/data/usb/PIONEER/ {{dst}}/PIONEER/ + rm -f {{dst}}/PIONEER/rekordbox/RBFLTR.DAT + +usb-pdb dst="/Volumes/MAPLE": + @test -d {{dst}}/PIONEER/rekordbox || (echo "no rekordbox dir at {{dst}}/PIONEER"; exit 1) + COPYFILE_DISABLE=1 cp maple-py/data/MAPLE/PIONEER/rekordbox/export.pdb {{dst}}/PIONEER/rekordbox/export.pdb usb-reset dst="/Volumes/MAPLE": @test -d {{dst}} || (echo "USB not mounted at {{dst}}"; exit 1) rm -rf {{dst}}/Contents {{dst}}/PIONEER COPYFILE_DISABLE=1 rsync -rtvh --no-perms --no-owner --no-group --modify-window=2 maple-py/data/MAPLE/Contents/ {{dst}}/Contents/ COPYFILE_DISABLE=1 rsync -rtvh --no-perms --no-owner --no-group --modify-window=2 maple-py/data/MAPLE/PIONEER/ {{dst}}/PIONEER/ - find {{dst}} -name '._*' -delete maple-setup: bash ./etc/maple_setup.sh diff --git a/maple-py/src/kaitai/rekordbox_anlz.py b/maple-py/src/kaitai/rekordbox_anlz.py new file mode 100644 index 0000000..41b57db --- /dev/null +++ b/maple-py/src/kaitai/rekordbox_anlz.py @@ -0,0 +1,897 @@ +# This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild +# type: ignore + +import kaitaistruct +from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from enum import IntEnum +import struct + + +if getattr(kaitaistruct, 'API_VERSION', (0, 9)) < (0, 11): + raise Exception("Incompatible Kaitai Struct Python API: 0.11 or later is required, but you have %s" % (kaitaistruct.__version__)) + +class RekordboxAnlz(KaitaiStruct): + """These files are created by rekordbox when analyzing audio tracks + to facilitate DJ performance. They include waveforms, beat grids + (information about the precise time at which each beat occurs), + time indices to allow efficient seeking to specific positions + inside variable bit-rate audio streams, and lists of memory cues + and loop points. They are used by Pioneer professional DJ + equipment. + + The format has been reverse-engineered to facilitate sophisticated + integrations with light and laser shows, videos, and other musical + instruments, by supporting deep knowledge of what is playing and + what is coming next through monitoring the network communications + of the players. + + .. seealso:: + Source - https://reverseengineering.stackexchange.com/questions/4311/help-reversing-a-edb-database-file-for-pioneers-rekordbox-software + """ + + class CueEntryStatus(IntEnum): + disabled = 0 + enabled = 1 + active_loop = 4 + + class CueEntryType(IntEnum): + memory_cue = 1 + loop = 2 + + class CueListType(IntEnum): + memory_cues = 0 + hot_cues = 1 + + class MoodHighPhrase(IntEnum): + intro = 1 + up = 2 + down = 3 + chorus = 5 + outro = 6 + + class MoodLowPhrase(IntEnum): + intro = 1 + verse_1 = 2 + verse_1b = 3 + verse_1c = 4 + verse_2 = 5 + verse_2b = 6 + verse_2c = 7 + bridge = 8 + chorus = 9 + outro = 10 + + class MoodMidPhrase(IntEnum): + intro = 1 + verse_1 = 2 + verse_2 = 3 + verse_3 = 4 + verse_4 = 5 + verse_5 = 6 + verse_6 = 7 + bridge = 8 + chorus = 9 + outro = 10 + + class SectionTags(IntEnum): + cues_2 = 1346588466 + cues = 1346588482 + path = 1347441736 + beat_grid = 1347507290 + song_structure = 1347638089 + vbr = 1347830354 + wave_preview = 1347895638 + wave_tiny = 1347900978 + wave_scroll = 1347900979 + wave_color_preview = 1347900980 + wave_color_scroll = 1347900981 + wave_3band_preview = 1347900982 + wave_3band_scroll = 1347900983 + + class TrackBank(IntEnum): + default = 0 + cool = 1 + natural = 2 + hot = 3 + subtle = 4 + warm = 5 + vivid = 6 + club_1 = 7 + club_2 = 8 + + class TrackMood(IntEnum): + high = 1 + mid = 2 + low = 3 + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz, self).__init__(_io) + self._parent = _parent + self._root = _root or self + self._read() + + def _read(self): + self.magic = self._io.read_bytes(4) + if not self.magic == b"\x50\x4D\x41\x49": + raise kaitaistruct.ValidationNotEqualError(b"\x50\x4D\x41\x49", self.magic, self._io, u"/seq/0") + self.len_header = self._io.read_u4be() + self.len_file = self._io.read_u4be() + self._unnamed3 = self._io.read_bytes(self.len_header - self._io.pos()) + self.sections = [] + i = 0 + while not self._io.is_eof(): + self.sections.append(RekordboxAnlz.TaggedSection(self._io, self, self._root)) + i += 1 + + + + def _fetch_instances(self): + pass + for i in range(len(self.sections)): + pass + self.sections[i]._fetch_instances() + + + class BeatGridBeat(KaitaiStruct): + """Describes an individual beat in a beat grid. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.BeatGridBeat, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.beat_number = self._io.read_u2be() + self.tempo = self._io.read_u2be() + self.time = self._io.read_u4be() + + + def _fetch_instances(self): + pass + + + class BeatGridTag(KaitaiStruct): + """Holds a list of all the beats found within the track, recording + their bar position, the time at which they occur, and the tempo + at that point. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.BeatGridTag, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self._unnamed0 = self._io.read_u4be() + self._unnamed1 = self._io.read_u4be() + self.num_beats = self._io.read_u4be() + self.beats = [] + for i in range(self.num_beats): + self.beats.append(RekordboxAnlz.BeatGridBeat(self._io, self, self._root)) + + + + def _fetch_instances(self): + pass + for i in range(len(self.beats)): + pass + self.beats[i]._fetch_instances() + + + + class CueEntry(KaitaiStruct): + """A cue list entry. Can either represent a memory cue or a loop. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.CueEntry, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.magic = self._io.read_bytes(4) + if not self.magic == b"\x50\x43\x50\x54": + raise kaitaistruct.ValidationNotEqualError(b"\x50\x43\x50\x54", self.magic, self._io, u"/types/cue_entry/seq/0") + self.len_header = self._io.read_u4be() + self.len_entry = self._io.read_u4be() + self.hot_cue = self._io.read_u4be() + self.status = KaitaiStream.resolve_enum(RekordboxAnlz.CueEntryStatus, self._io.read_u4be()) + self._unnamed5 = self._io.read_u4be() + self.order_first = self._io.read_u2be() + self.order_last = self._io.read_u2be() + self.type = KaitaiStream.resolve_enum(RekordboxAnlz.CueEntryType, self._io.read_u1()) + self._unnamed9 = self._io.read_bytes(3) + self.time = self._io.read_u4be() + self.loop_time = self._io.read_u4be() + self._unnamed12 = self._io.read_bytes(16) + + + def _fetch_instances(self): + pass + + + class CueExtendedEntry(KaitaiStruct): + """A cue extended list entry. Can either describe a memory cue or a + loop. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.CueExtendedEntry, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.magic = self._io.read_bytes(4) + if not self.magic == b"\x50\x43\x50\x32": + raise kaitaistruct.ValidationNotEqualError(b"\x50\x43\x50\x32", self.magic, self._io, u"/types/cue_extended_entry/seq/0") + self.len_header = self._io.read_u4be() + self.len_entry = self._io.read_u4be() + self.hot_cue = self._io.read_u4be() + self.type = KaitaiStream.resolve_enum(RekordboxAnlz.CueEntryType, self._io.read_u1()) + self._unnamed5 = self._io.read_bytes(3) + self.time = self._io.read_u4be() + self.loop_time = self._io.read_u4be() + self.color_id = self._io.read_u1() + self._unnamed9 = self._io.read_bytes(7) + self.loop_numerator = self._io.read_u2be() + self.loop_denominator = self._io.read_u2be() + if self.len_entry > 43: + pass + self.len_comment = self._io.read_u4be() + + if self.len_entry > 43: + pass + self.comment = (self._io.read_bytes(self.len_comment)).decode(u"UTF-16BE") + + if self.len_entry - self.len_comment > 44: + pass + self.color_code = self._io.read_u1() + + if self.len_entry - self.len_comment > 45: + pass + self.color_red = self._io.read_u1() + + if self.len_entry - self.len_comment > 46: + pass + self.color_green = self._io.read_u1() + + if self.len_entry - self.len_comment > 47: + pass + self.color_blue = self._io.read_u1() + + if self.len_entry - self.len_comment > 48: + pass + self._unnamed18 = self._io.read_bytes((self.len_entry - 48) - self.len_comment) + + + + def _fetch_instances(self): + pass + if self.len_entry > 43: + pass + + if self.len_entry > 43: + pass + + if self.len_entry - self.len_comment > 44: + pass + + if self.len_entry - self.len_comment > 45: + pass + + if self.len_entry - self.len_comment > 46: + pass + + if self.len_entry - self.len_comment > 47: + pass + + if self.len_entry - self.len_comment > 48: + pass + + + + class CueExtendedTag(KaitaiStruct): + """A variation of cue_tag which was introduced with the nxs2 line, + and adds descriptive names. (Still comes in two forms, either + holding memory cues and loop points, or holding hot cues and + loop points.) Also includes hot cues D through H and color assignment. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.CueExtendedTag, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.type = KaitaiStream.resolve_enum(RekordboxAnlz.CueListType, self._io.read_u4be()) + self.num_cues = self._io.read_u2be() + self._unnamed2 = self._io.read_bytes(2) + self.cues = [] + for i in range(self.num_cues): + self.cues.append(RekordboxAnlz.CueExtendedEntry(self._io, self, self._root)) + + + + def _fetch_instances(self): + pass + for i in range(len(self.cues)): + pass + self.cues[i]._fetch_instances() + + + + class CueTag(KaitaiStruct): + """Stores either a list of ordinary memory cues and loop points, or + a list of hot cues and loop points. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.CueTag, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.type = KaitaiStream.resolve_enum(RekordboxAnlz.CueListType, self._io.read_u4be()) + self._unnamed1 = self._io.read_bytes(2) + self.num_cues = self._io.read_u2be() + self.memory_count = self._io.read_u4be() + self.cues = [] + for i in range(self.num_cues): + self.cues.append(RekordboxAnlz.CueEntry(self._io, self, self._root)) + + + + def _fetch_instances(self): + pass + for i in range(len(self.cues)): + pass + self.cues[i]._fetch_instances() + + + + class PathTag(KaitaiStruct): + """Stores the file path of the audio file to which this analysis + applies. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.PathTag, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.len_path = self._io.read_u4be() + if self.len_path > 1: + pass + self.path = (self._io.read_bytes(self.len_path - 2)).decode(u"UTF-16BE") + + + + def _fetch_instances(self): + pass + if self.len_path > 1: + pass + + + + class PhraseHigh(KaitaiStruct): + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.PhraseHigh, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.id = KaitaiStream.resolve_enum(RekordboxAnlz.MoodHighPhrase, self._io.read_u2be()) + + + def _fetch_instances(self): + pass + + + class PhraseLow(KaitaiStruct): + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.PhraseLow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.id = KaitaiStream.resolve_enum(RekordboxAnlz.MoodLowPhrase, self._io.read_u2be()) + + + def _fetch_instances(self): + pass + + + class PhraseMid(KaitaiStruct): + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.PhraseMid, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.id = KaitaiStream.resolve_enum(RekordboxAnlz.MoodMidPhrase, self._io.read_u2be()) + + + def _fetch_instances(self): + pass + + + class SongStructureBody(KaitaiStruct): + """Stores the rest of the song structure tag, which can only be + parsed after unmasking. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.SongStructureBody, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.mood = KaitaiStream.resolve_enum(RekordboxAnlz.TrackMood, self._io.read_u2be()) + self._unnamed1 = self._io.read_bytes(6) + self.end_beat = self._io.read_u2be() + self._unnamed3 = self._io.read_bytes(2) + self.raw_bank = self._io.read_u1() + self._unnamed5 = self._io.read_bytes(1) + self.entries = [] + for i in range(self._parent.len_entries): + self.entries.append(RekordboxAnlz.SongStructureEntry(self._io, self, self._root)) + + + + def _fetch_instances(self): + pass + for i in range(len(self.entries)): + pass + self.entries[i]._fetch_instances() + + + @property + def bank(self): + """The stylistic bank which can be assigned to the track in rekordbox Lighting mode, if raw_bank has a legal value. + """ + if hasattr(self, '_m_bank'): + return self._m_bank + + if self.raw_bank < 9: + pass + self._m_bank = KaitaiStream.resolve_enum(RekordboxAnlz.TrackBank, self.raw_bank) + + return getattr(self, '_m_bank', None) + + + class SongStructureEntry(KaitaiStruct): + """A song structure entry, represents a single phrase. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.SongStructureEntry, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.index = self._io.read_u2be() + self.beat = self._io.read_u2be() + _on = self._parent.mood + if _on == RekordboxAnlz.TrackMood.high: + pass + self.kind = RekordboxAnlz.PhraseHigh(self._io, self, self._root) + elif _on == RekordboxAnlz.TrackMood.low: + pass + self.kind = RekordboxAnlz.PhraseLow(self._io, self, self._root) + elif _on == RekordboxAnlz.TrackMood.mid: + pass + self.kind = RekordboxAnlz.PhraseMid(self._io, self, self._root) + else: + pass + self.kind = RekordboxAnlz.PhraseMid(self._io, self, self._root) + self._unnamed3 = self._io.read_bytes(1) + self.k1 = self._io.read_u1() + self._unnamed5 = self._io.read_bytes(1) + self.k2 = self._io.read_u1() + self._unnamed7 = self._io.read_bytes(1) + self.b = self._io.read_u1() + self.beat2 = self._io.read_u2be() + self.beat3 = self._io.read_u2be() + self.beat4 = self._io.read_u2be() + self._unnamed12 = self._io.read_bytes(1) + self.k3 = self._io.read_u1() + self._unnamed14 = self._io.read_bytes(1) + self.fill = self._io.read_u1() + self.beat_fill = self._io.read_u2be() + + + def _fetch_instances(self): + pass + _on = self._parent.mood + if _on == RekordboxAnlz.TrackMood.high: + pass + self.kind._fetch_instances() + elif _on == RekordboxAnlz.TrackMood.low: + pass + self.kind._fetch_instances() + elif _on == RekordboxAnlz.TrackMood.mid: + pass + self.kind._fetch_instances() + else: + pass + self.kind._fetch_instances() + + + class SongStructureTag(KaitaiStruct): + """Stores the song structure, also known as phrases (intro, verse, + bridge, chorus, up, down, outro). + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.SongStructureTag, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.len_entry_bytes = self._io.read_u4be() + self.len_entries = self._io.read_u2be() + self._raw__raw_body = self._io.read_bytes_full() + self._raw_body = KaitaiStream.process_xor_many(self._raw__raw_body, (self.mask if self.is_masked else b"\x00")) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.SongStructureBody(_io__raw_body, self, self._root) + + + def _fetch_instances(self): + pass + self.body._fetch_instances() + _ = self.raw_mood + if hasattr(self, '_m_raw_mood'): + pass + + + @property + def c(self): + if hasattr(self, '_m_c'): + return self._m_c + + self._m_c = self.len_entries + return getattr(self, '_m_c', None) + + @property + def is_masked(self): + if hasattr(self, '_m_is_masked'): + return self._m_is_masked + + self._m_is_masked = self.raw_mood > 20 + return getattr(self, '_m_is_masked', None) + + @property + def mask(self): + if hasattr(self, '_m_mask'): + return self._m_mask + + self._m_mask = struct.pack('19B', (203 + self.c), (225 + self.c), (238 + self.c), (250 + self.c), (229 + self.c), (238 + self.c), (173 + self.c), (238 + self.c), (233 + self.c), (210 + self.c), (233 + self.c), (235 + self.c), (225 + self.c), (233 + self.c), (243 + self.c), (232 + self.c), (233 + self.c), (244 + self.c), (225 + self.c)) + return getattr(self, '_m_mask', None) + + @property + def raw_mood(self): + """This is a way to tell whether the rest of the tag has been masked. The value is supposed + to range from 1 to 3, but in masked files it will be much larger. + """ + if hasattr(self, '_m_raw_mood'): + return self._m_raw_mood + + _pos = self._io.pos() + self._io.seek(6) + self._m_raw_mood = self._io.read_u2be() + self._io.seek(_pos) + return getattr(self, '_m_raw_mood', None) + + + class TaggedSection(KaitaiStruct): + """A type-tagged file section, identified by a four-byte magic + sequence, with a header specifying its length, and whose payload + is determined by the type tag. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.TaggedSection, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.fourcc = KaitaiStream.resolve_enum(RekordboxAnlz.SectionTags, self._io.read_s4be()) + self.len_header = self._io.read_u4be() + self.len_tag = self._io.read_u4be() + _on = self.fourcc + if _on == RekordboxAnlz.SectionTags.beat_grid: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.BeatGridTag(_io__raw_body, self, self._root) + elif _on == RekordboxAnlz.SectionTags.cues: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.CueTag(_io__raw_body, self, self._root) + elif _on == RekordboxAnlz.SectionTags.cues_2: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.CueExtendedTag(_io__raw_body, self, self._root) + elif _on == RekordboxAnlz.SectionTags.path: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.PathTag(_io__raw_body, self, self._root) + elif _on == RekordboxAnlz.SectionTags.song_structure: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.SongStructureTag(_io__raw_body, self, self._root) + elif _on == RekordboxAnlz.SectionTags.vbr: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.VbrTag(_io__raw_body, self, self._root) + elif _on == RekordboxAnlz.SectionTags.wave_3band_preview: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.Wave3bandPreviewTag(_io__raw_body, self, self._root) + elif _on == RekordboxAnlz.SectionTags.wave_3band_scroll: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.Wave3bandScrollTag(_io__raw_body, self, self._root) + elif _on == RekordboxAnlz.SectionTags.wave_color_preview: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.WaveColorPreviewTag(_io__raw_body, self, self._root) + elif _on == RekordboxAnlz.SectionTags.wave_color_scroll: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.WaveColorScrollTag(_io__raw_body, self, self._root) + elif _on == RekordboxAnlz.SectionTags.wave_preview: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.WavePreviewTag(_io__raw_body, self, self._root) + elif _on == RekordboxAnlz.SectionTags.wave_scroll: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.WaveScrollTag(_io__raw_body, self, self._root) + elif _on == RekordboxAnlz.SectionTags.wave_tiny: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.WavePreviewTag(_io__raw_body, self, self._root) + else: + pass + self._raw_body = self._io.read_bytes(self.len_tag - 12) + _io__raw_body = KaitaiStream(BytesIO(self._raw_body)) + self.body = RekordboxAnlz.UnknownTag(_io__raw_body, self, self._root) + + + def _fetch_instances(self): + pass + _on = self.fourcc + if _on == RekordboxAnlz.SectionTags.beat_grid: + pass + self.body._fetch_instances() + elif _on == RekordboxAnlz.SectionTags.cues: + pass + self.body._fetch_instances() + elif _on == RekordboxAnlz.SectionTags.cues_2: + pass + self.body._fetch_instances() + elif _on == RekordboxAnlz.SectionTags.path: + pass + self.body._fetch_instances() + elif _on == RekordboxAnlz.SectionTags.song_structure: + pass + self.body._fetch_instances() + elif _on == RekordboxAnlz.SectionTags.vbr: + pass + self.body._fetch_instances() + elif _on == RekordboxAnlz.SectionTags.wave_3band_preview: + pass + self.body._fetch_instances() + elif _on == RekordboxAnlz.SectionTags.wave_3band_scroll: + pass + self.body._fetch_instances() + elif _on == RekordboxAnlz.SectionTags.wave_color_preview: + pass + self.body._fetch_instances() + elif _on == RekordboxAnlz.SectionTags.wave_color_scroll: + pass + self.body._fetch_instances() + elif _on == RekordboxAnlz.SectionTags.wave_preview: + pass + self.body._fetch_instances() + elif _on == RekordboxAnlz.SectionTags.wave_scroll: + pass + self.body._fetch_instances() + elif _on == RekordboxAnlz.SectionTags.wave_tiny: + pass + self.body._fetch_instances() + else: + pass + self.body._fetch_instances() + + + class UnknownTag(KaitaiStruct): + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.UnknownTag, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + pass + + + def _fetch_instances(self): + pass + + + class VbrTag(KaitaiStruct): + """Stores an index allowing rapid seeking to particular times + within a variable-bitrate audio file. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.VbrTag, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self._unnamed0 = self._io.read_u4be() + self.index = [] + for i in range(400): + self.index.append(self._io.read_u4be()) + + + + def _fetch_instances(self): + pass + for i in range(len(self.index)): + pass + + + + class Wave3bandPreviewTag(KaitaiStruct): + """The minimalist CDJ-3000 waveform preview image suitable for display + above the touch strip for jumping to a track position. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.Wave3bandPreviewTag, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.len_entry_bytes = self._io.read_u4be() + self.len_entries = self._io.read_u4be() + self.entries = self._io.read_bytes(self.len_entries * self.len_entry_bytes) + + + def _fetch_instances(self): + pass + + + class Wave3bandScrollTag(KaitaiStruct): + """The minimalist CDJ-3000 waveform image suitable for scrolling along + as a track plays on newer high-resolution hardware. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.Wave3bandScrollTag, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.len_entry_bytes = self._io.read_u4be() + self.len_entries = self._io.read_u4be() + self._unnamed2 = self._io.read_u4be() + self.entries = self._io.read_bytes(self.len_entries * self.len_entry_bytes) + + + def _fetch_instances(self): + pass + + + class WaveColorPreviewTag(KaitaiStruct): + """A larger, colorful waveform preview image suitable for display + above the touch strip for jumping to a track position on newer + high-resolution players. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.WaveColorPreviewTag, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.len_entry_bytes = self._io.read_u4be() + self.len_entries = self._io.read_u4be() + self._unnamed2 = self._io.read_u4be() + self.entries = self._io.read_bytes(self.len_entries * self.len_entry_bytes) + + + def _fetch_instances(self): + pass + + + class WaveColorScrollTag(KaitaiStruct): + """A larger, colorful waveform image suitable for scrolling along + as a track plays on newer high-resolution hardware. Also + contains a higher-resolution blue/white waveform. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.WaveColorScrollTag, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.len_entry_bytes = self._io.read_u4be() + self.len_entries = self._io.read_u4be() + self._unnamed2 = self._io.read_u4be() + self.entries = self._io.read_bytes(self.len_entries * self.len_entry_bytes) + + + def _fetch_instances(self): + pass + + + class WavePreviewTag(KaitaiStruct): + """Stores a waveform preview image suitable for display above + the touch strip for jumping to a track position. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.WavePreviewTag, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.len_data = self._io.read_u4be() + self._unnamed1 = self._io.read_u4be() + if self._parent.len_tag > self._parent.len_header: + pass + self.data = self._io.read_bytes(self.len_data) + + + + def _fetch_instances(self): + pass + if self._parent.len_tag > self._parent.len_header: + pass + + + + class WaveScrollTag(KaitaiStruct): + """A larger waveform image suitable for scrolling along as a track + plays. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxAnlz.WaveScrollTag, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.len_entry_bytes = self._io.read_u4be() + self.len_entries = self._io.read_u4be() + self._unnamed2 = self._io.read_u4be() + self.entries = self._io.read_bytes(self.len_entries * self.len_entry_bytes) + + + def _fetch_instances(self): + pass + + + diff --git a/maple-py/src/kaitai/rekordbox_pdb.py b/maple-py/src/kaitai/rekordbox_pdb.py index 30913e2..eb7c5e7 100644 --- a/maple-py/src/kaitai/rekordbox_pdb.py +++ b/maple-py/src/kaitai/rekordbox_pdb.py @@ -70,8 +70,8 @@ class RekordboxPdb(KaitaiStruct): unknown_14 = 14 unknown_15 = 15 columns = 16 - unknown_17 = 17 - unknown_18 = 18 + browse_filter_index = 17 + browse_menu = 18 history = 19 class PageTypeExt(IntEnum): @@ -259,6 +259,62 @@ class RekordboxPdb(KaitaiStruct): self.path._fetch_instances() + class BrowseFilterIndexRow(KaitaiStruct): + """A row in the `browse_filter_index` (was `unknown_17`) table. Lists the columns + that appear in the player's "Filter by ->" sidebar, in display order. The + reverse-engineered semantics: each row enumerates one filterable column. The + first group (`d == 0`) has 12 rows that look like type/icon descriptors; the + second group (`d == 1..10`) defines the navigation entries with `d` as the + sort position. The 22 column ids that appear here are exactly the + "filterable" subset of the 27 columns -- CUE, COMMENTS, DJ PLAY COUNT, + DEFAULT, ALPHABET do not appear. Verified byte-for-byte identical across + multiple working rekordbox exports. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.BrowseFilterIndexRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.column_id = self._io.read_u2le() + self.secondary_column = self._io.read_u2le() + self.filter_type = self._io.read_u2le() + self.sort_index = self._io.read_u2le() + + + def _fetch_instances(self): + pass + + + class BrowseMenuRow(KaitaiStruct): + """A row in the `browse_menu` (was `unknown_18`) table. Together with + `browse_filter_index`, this drives the player's top-level browse navigation. + Semantic model: `display_column` is the column shown in the menu; + `sort_column` is the column used to sort/key it. When the two are equal the + entry opens a drilldown of that column's distinct values (e.g. + `(ARTIST, ARTIST)` -> list of artists). When they differ the entry opens a + flat list of `display_column` rows sorted by `sort_column` (e.g. + `(BPM, TRACK)` -> tracks sorted by BPM). Verified content is identical + across all observed working exports. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.BrowseMenuRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.sort_column = self._io.read_u2le() + self.display_column = self._io.read_u2le() + self.flags = self._io.read_u2le() + self.reserved = self._io.read_u2le() + + + def _fetch_instances(self): + pass + + class ColorRow(KaitaiStruct): """A row that holds a color name and the associated ID. """ @@ -811,6 +867,12 @@ class RekordboxPdb(KaitaiStruct): elif _on == RekordboxPdb.PageType.artwork: pass self._m_body._fetch_instances() + elif _on == RekordboxPdb.PageType.browse_filter_index: + pass + self._m_body._fetch_instances() + elif _on == RekordboxPdb.PageType.browse_menu: + pass + self._m_body._fetch_instances() elif _on == RekordboxPdb.PageType.colors: pass self._m_body._fetch_instances() @@ -876,6 +938,12 @@ class RekordboxPdb(KaitaiStruct): elif _on == RekordboxPdb.PageType.artwork: pass self._m_body = RekordboxPdb.ArtworkRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageType.browse_filter_index: + pass + self._m_body = RekordboxPdb.BrowseFilterIndexRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageType.browse_menu: + pass + self._m_body = RekordboxPdb.BrowseMenuRow(self._io, self, self._root) elif _on == RekordboxPdb.PageType.colors: pass self._m_body = RekordboxPdb.ColorRow(self._io, self, self._root) diff --git a/maple-py/src/maple/anlz.py b/maple-py/src/maple/anlz.py new file mode 100644 index 0000000..87bf47a --- /dev/null +++ b/maple-py/src/maple/anlz.py @@ -0,0 +1,41 @@ +"""Generate a minimal rekordbox ANLZ0000.DAT analysis file. + +The CDJ/XDJ reads the path from a `PPTH` section to confirm which audio +file the analysis applies to. We emit just that section -- no beat grid, +no waveforms -- which is enough for the player to consider the track's +analysis "present" and surface the track in the library. Without an +analyze file at the path the pdb references, some firmware hides the +track entirely. + +All multi-byte fields in ANLZ files are BIG-endian, unlike the pdb. +""" +from pathlib import Path +from struct import pack + + +def _section_pth(path: str) -> bytes: + """PPTH section: u4 len_path, then UTF-16BE bytes (null-terminated).""" + utf16 = path.encode("utf-16-be") + b"\x00\x00" + len_path = len(utf16) + body = pack(">I", len_path) + utf16 + # Section header: 4 fourcc + u4 len_header + u4 len_tag + fourcc = b"PPTH" + len_header = 16 # section header size (incl. fourcc + 2 u4s + len_path u4) + len_tag = 12 + len(body) # full section size + return fourcc + pack(">II", len_header, len_tag) + body + + +def build_minimal_anlz(audio_path: str) -> bytes: + """Return bytes of a minimal ANLZ0000.DAT pointing at `audio_path`.""" + pth = _section_pth(audio_path) + header_len = 28 + file_len = header_len + len(pth) + header = b"PMAI" + pack(">II", header_len, file_len) + header += b"\x00" * (header_len - len(header)) + assert len(header) == header_len + return header + pth + + +def write_anlz(audio_path: str, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(build_minimal_anlz(audio_path)) diff --git a/maple-py/src/maple/cli.py b/maple-py/src/maple/cli.py index f6c5e10..58536fa 100644 --- a/maple-py/src/maple/cli.py +++ b/maple-py/src/maple/cli.py @@ -6,21 +6,23 @@ from concurrent.futures import ThreadPoolExecutor from datetime import date from pathlib import Path +from maple.anlz import write_anlz from maple.artwork import extract_artwork from maple.convert import convert_flac_to_mp3, verify_duration from maple.copy import stage_audio, staged_path from maple.meta import read_tags -from maple.model import Album, Artist, Artwork, Database, PlaylistEntry, PlaylistNode, Track +from maple.model import Album, Artist, Artwork, Database, Genre, PlaylistEntry, PlaylistNode, Track from maple.pdb.writer import write_pdb from maple.scan import scan_audio, scan_playlists -def build(input_dir: Path, usb_root: Path) -> Database: +def build(input_dir: Path, usb_root: Path, with_playlists: bool = True) -> Database: db = Database() today = date.today().isoformat() artist_index: dict[str, int] = {} album_index: dict[tuple[int, str], int] = {} + genre_index: dict[str, int] = {} track_by_src: dict[Path, int] = {} artwork_cache: dict[str, int] = {} @@ -70,7 +72,25 @@ def build(input_dir: Path, usb_root: Path) -> Database: if art_file_path is not None: db.artwork.append(Artwork(id=artwork_id, path=art_file_path)) + genre_id = 0 + if meta.genre: + genre_id = genre_index.get(meta.genre, 0) + if genre_id == 0: + genre_id = len(db.genres) + 1 + genre_index[meta.genre] = genre_id + db.genres.append(Genre(id=genre_id, name=meta.genre)) + track_id = len(db.tracks) + 1 + # Reference exports always populate analyze_path even when the + # ANLZ files are real; tracks with an empty analyze_path appear + # to be hidden from the player's library. Synthesise a stable + # path keyed off the track id even if we don't write the .DAT. + import zlib + h = zlib.crc32(file_path.encode()) & 0xFFFFFFFF + bucket = f"P{(h >> 20) & 0xFFF:03X}" + analyze_id = f"{h & 0xFFFFFFFF:08X}" + analyze_path = f"/PIONEER/USBANLZ/{bucket}/{analyze_id}/ANLZ0000.DAT" + write_anlz(file_path, usb_root / analyze_path.lstrip("/")) db.tracks.append(Track( id=track_id, title=meta.title, @@ -85,12 +105,18 @@ def build(input_dir: Path, usb_root: Path) -> Database: duration=meta.duration, track_number=meta.track_number, year=meta.year, + tempo=meta.tempo, date_added=today, + analyze_path=analyze_path, + analyze_date=today, artwork_id=artwork_id, + genre_id=genre_id, source_path=original, )) track_by_src[original.resolve()] = track_id + if not with_playlists: + return db playlists = scan_playlists(input_dir) for sort_order, (name, paths) in enumerate(playlists.items(), start=1): playlist_id = len(db.playlist_nodes) + 1 @@ -119,12 +145,13 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="maple", description="Build a rekordbox USB from a folder of audio + m3u8 playlists.") parser.add_argument("input_dir", nargs="?", type=Path, default=Path("./data/input")) parser.add_argument("output_root", nargs="?", type=Path, default=Path("./data/usb")) + parser.add_argument("--no-playlists", action="store_true", help="Skip m3u8 playlist scanning") args = parser.parse_args(argv) if not args.input_dir.is_dir(): parser.error(f"input_dir {args.input_dir} is not a directory") - db = build(args.input_dir, args.output_root) + db = build(args.input_dir, args.output_root, with_playlists=not args.no_playlists) pdb_path = args.output_root / "PIONEER" / "rekordbox" / "export.pdb" write_pdb(db, pdb_path) diff --git a/maple-py/src/maple/meta.py b/maple-py/src/maple/meta.py index 9d65e29..8cfc907 100644 --- a/maple-py/src/maple/meta.py +++ b/maple-py/src/maple/meta.py @@ -5,11 +5,15 @@ from pathlib import Path from mutagen import File as MutagenFile +DEFAULT_TEMPO_X100 = 12000 # 120.00 BPM, used when no BPM tag is present + + @dataclass class TrackMeta: title: str artist: str album: str + genre: str track_number: int year: int duration: int @@ -17,6 +21,7 @@ class TrackMeta: sample_rate: int sample_depth: int file_size: int + tempo: int = DEFAULT_TEMPO_X100 # BPM * 100 (rekordbox storage format) def _first(value) -> str: @@ -39,6 +44,17 @@ def _int(value) -> int: return 0 +def _tempo_x100(value) -> int: + """Parse a BPM tag into integer BPM * 100. 0 means "not present".""" + s = _first(value).strip() + if not s: + return 0 + try: + return int(round(float(s) * 100)) + except ValueError: + return 0 + + def read_tags(path: Path) -> TrackMeta: file_size = path.stat().st_size audio = MutagenFile(path, easy=True) @@ -47,6 +63,7 @@ def read_tags(path: Path) -> TrackMeta: title=path.stem, artist="Unknown Artist", album="Unknown Album", + genre="", track_number=0, year=0, duration=0, @@ -60,8 +77,10 @@ def read_tags(path: Path) -> TrackMeta: title = _first(tags.get("title")) or path.stem artist = _first(tags.get("artist")) or "Unknown Artist" album = _first(tags.get("album")) or "Unknown Album" + genre = _first(tags.get("genre")) track_number = _int(tags.get("tracknumber")) year = _int(tags.get("date") or tags.get("year")) + tempo = _tempo_x100(tags.get("bpm")) or DEFAULT_TEMPO_X100 info = getattr(audio, "info", None) duration = int(getattr(info, "length", 0) or 0) @@ -74,6 +93,7 @@ def read_tags(path: Path) -> TrackMeta: title=title, artist=artist, album=album, + genre=genre, track_number=track_number, year=year, duration=duration, @@ -81,4 +101,5 @@ def read_tags(path: Path) -> TrackMeta: sample_rate=sample_rate, sample_depth=sample_depth, file_size=file_size, + tempo=tempo, ) diff --git a/maple-py/src/maple/model.py b/maple-py/src/maple/model.py index 76bf551..5b81289 100644 --- a/maple-py/src/maple/model.py +++ b/maple-py/src/maple/model.py @@ -16,6 +16,18 @@ class Album: artist_id: int +@dataclass +class Genre: + id: int + name: str + + +@dataclass +class Color: + id: int + name: str + + @dataclass class Track: id: int @@ -40,6 +52,7 @@ class Track: analyze_path: str = "" analyze_date: str = "" artwork_id: int = 0 + genre_id: int = 0 source_path: Path | None = None @@ -90,6 +103,41 @@ DEFAULT_COLUMNS: list[Column] = [ ] +# Reference-derived rows for the un-decoded `unknown_17` and `unknown_18` +# tables. Each tuple is (a, b, c, d) packed as four little-endian u2s. +# These appear to drive the player's "Filter by ->" menu (which columns +# are filterable, sort order, filter type code). Speculative purpose; +# verified to match the reference byte-for-byte. +UNKNOWN_17_ROWS: list[tuple[int, int, int, int]] = [ + (1, 1, 355, 0), (5, 6, 261, 0), (6, 7, 355, 0), (7, 8, 355, 0), + (8, 9, 355, 0), (9, 10, 355, 0), (10, 11, 355, 0), (13, 15, 355, 0), + (14, 19, 260, 0), (15, 20, 262, 0), (16, 21, 355, 0), (18, 23, 355, 0), + (2, 2, 2, 1), (3, 3, 3, 2), (4, 4, 1, 3), (11, 12, 99, 4), + (17, 5, 99, 5), (19, 22, 99, 6), (20, 18, 99, 7), (27, 26, 611, 8), + (24, 17, 99, 9), (22, 27, 99, 10), +] +UNKNOWN_18_ROWS: list[tuple[int, int, int, int]] = [ + (1, 6, 1, 0), (21, 7, 1, 0), (14, 8, 1, 0), (8, 9, 1, 0), + (9, 10, 1, 0), (10, 11, 1, 0), (15, 13, 1, 0), (13, 15, 1, 0), + (23, 16, 1, 0), (22, 17, 1, 0), (25, 0, 256, 0), (26, 1, 512, 0), + (2, 2, 768, 0), (3, 3, 1024, 0), (5, 4, 1282, 0), (6, 5, 1536, 0), + (11, 12, 1792, 0), +] + + +# The standard rekordbox color label palette. +STANDARD_COLORS: list[Color] = [ + Color(1, "Pink"), + Color(2, "Red"), + Color(3, "Orange"), + Color(4, "Yellow"), + Color(5, "Green"), + Color(6, "Aqua"), + Color(7, "Blue"), + Color(8, "Purple"), +] + + @dataclass class PlaylistNode: id: int @@ -114,3 +162,5 @@ class Database: playlist_nodes: list[PlaylistNode] = field(default_factory=list) playlist_entries: list[PlaylistEntry] = field(default_factory=list) artwork: list[Artwork] = field(default_factory=list) + genres: list[Genre] = field(default_factory=list) + colors: list[Color] = field(default_factory=list) diff --git a/maple-py/src/maple/pdb/pager.py b/maple-py/src/maple/pdb/pager.py index a752a2b..2fa5dec 100644 --- a/maple-py/src/maple/pdb/pager.py +++ b/maple-py/src/maple/pdb/pager.py @@ -41,10 +41,11 @@ def build_strange_page(page_index: int, page_type: int, next_page: int) -> bytes class PageBuilder: - def __init__(self, page_index: int, page_type: int): + def __init__(self, page_index: int, page_type: int, max_rows: int = 0): self.page_index = page_index self.page_type = page_type self.next_page = page_index # caller patches before finalize + self.max_rows = max_rows self.row_bytes: list[bytes] = [] self.row_offsets: list[int] = [] # relative to heap_pos self.heap_cursor = 0 # position relative to heap_pos @@ -55,6 +56,8 @@ class PageBuilder: footer_after = _num_row_groups(len(self.row_bytes) + 1) * ROW_GROUP_SIZE if end + HEADER_SIZE + footer_after > PAGE_SIZE: return False + if self.max_rows and len(self.row_bytes) >= self.max_rows: + return False self.row_offsets.append(aligned) self.row_bytes.append(row) self.heap_cursor = end @@ -73,9 +76,11 @@ class PageBuilder: if free_size < 0: raise ValueError("page overflow") - # Reference exports use 0x24 for data pages with rows. Empty data - # pages (post-strange) keep 0x44 to mark them as having no rows. - page_flags = 0x24 if n > 0 else 0x44 + # 0x00 for data pages with rows, 0x44 for empty ones. The maple-rs + # writer uses this pair and produces pdbs that load on the player; + # rekordbox-exported references use 0x24 which also works, so both + # are valid -- pick the simpler form. + page_flags = 0x00 if n > 0 else 0x44 header = b"" header += pack(" bytes: prefix += pack(" bytes: prefix += pack(" bytes: return _pad2(row) +def serialize_genre(g: Genre) -> bytes: + row = pack(" bytes: + """history_playlist_row: u4 id, devicesql name.""" + return _pad2(pack(" bytes: + """history_entry_row: u4 track_id, u4 playlist_id, u4 entry_index.""" + return pack(" bytes: + """The `history` table row format isn't decoded by the kaitai schema. + A 40-byte blob copied from the reference, with the date substituted. + """ + date_str = encode_devicesql(date_added) # short ASCII "2026-05-26" + bucket = encode_devicesql("1000") + empty = encode_devicesql("") + head = pack(" bytes: + # Schema: 5 bytes padding, u2 id, u1 (zero), then name string. + # The reference puts the low byte of id at byte 4 (last of the padding). + row = b"\x00\x00\x00\x00" + bytes([c.id & 0xFF]) + pack(" bytes: """A column row is: u2 id, u2 code, then a 'long' devicesql string whose UTF-16 body is wrapped by the sentinel codepoints U+FFFA / U+FFFB. diff --git a/maple-py/src/maple/pdb/writer.py b/maple-py/src/maple/pdb/writer.py index 6bfb9f5..c2e1b0f 100644 --- a/maple-py/src/maple/pdb/writer.py +++ b/maple-py/src/maple/pdb/writer.py @@ -2,13 +2,24 @@ from pathlib import Path from struct import pack -from maple.model import DEFAULT_COLUMNS, Database +from maple.model import ( + DEFAULT_COLUMNS, + STANDARD_COLORS, + UNKNOWN_17_ROWS, + UNKNOWN_18_ROWS, + Database, +) from maple.pdb.pager import PAGE_SIZE, PageBuilder, build_strange_page from maple.pdb.rows import ( serialize_album, serialize_artist, serialize_artwork, + serialize_color, serialize_column, + serialize_genre, + serialize_history_blob, + serialize_history_entry, + serialize_history_playlist, serialize_playlist_entry, serialize_playlist_tree, serialize_track, @@ -44,20 +55,17 @@ TABLE_ENTRY_SIZE = 16 # page_type,empty_candidate,first_page,last_page = 4*u4 def _build_chain(page_type: int, row_iter, start_index: int) -> tuple[list[bytes], int, int]: """Return (page_bytes_list, first_page, last_page). - Every table chain begins with a "strange" placeholder page (flags=0x64) - mirroring rekordbox's own exports. Data pages with rows follow. - Tables with no rows emit only the strange page. + Every chain begins with a "strange" placeholder page (flags=0x64) + mirroring the reference; data pages with rows follow. Tables with no + rows emit only the strange page. """ rows = list(row_iter) out: list[bytes] = [] - # Reserve start_index for the strange page; data pages follow. if not rows: - # Strange page only: it points to itself (end of chain). out.append(build_strange_page(start_index, page_type, next_page=start_index)) return out, start_index, start_index - # Build data pages starting at start_index + 1. data_pages: list[PageBuilder] = [] page_idx = start_index + 1 current = PageBuilder(page_idx, page_type) @@ -70,10 +78,7 @@ def _build_chain(page_type: int, row_iter, start_index: int) -> tuple[list[bytes raise ValueError(f"row of size {len(row)} too large for an empty page") data_pages.append(current) - # Strange page links to the first data page. out.append(build_strange_page(start_index, page_type, next_page=start_index + 1)) - - # Patch each data page's next_page; last points to itself. for i, p in enumerate(data_pages): if i == len(data_pages) - 1: p.next_page = p.page_index @@ -96,20 +101,42 @@ def write_pdb(db: Database, path: str | Path) -> None: serialized_pl_entries = [serialize_playlist_entry(e) for e in db.playlist_entries] serialized_artwork = [serialize_artwork(a) for a in db.artwork] serialized_columns = [serialize_column(c) for c in DEFAULT_COLUMNS] + serialized_genres = [serialize_genre(g) for g in db.genres] + # If the caller didn't populate colors, always emit the standard palette + # so the player's Color filter has a non-empty value set. + colors = db.colors if db.colors else STANDARD_COLORS + serialized_colors = [serialize_color(c) for c in colors] + # unknown_17/unknown_18 row format isn't decoded by the kaitai schema; + # we emit the reference's byte-for-byte rows (four little-endian u2s). + serialized_unknown_17 = [pack("