diff --git a/maple-py/compare_pdb.py b/maple-py/compare_pdb.py new file mode 100644 index 0000000..a3ff9c1 --- /dev/null +++ b/maple-py/compare_pdb.py @@ -0,0 +1,57 @@ +"""Parse two .pdb files with the kaitai schema and summarise differences.""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent / "src")) +from io import BytesIO +from kaitaistruct import KaitaiStream +from kaitai.rekordbox_pdb import RekordboxPdb + + +PAGE_TYPE_NAMES = {t.value: t.name for t in RekordboxPdb.PageType} + + +def summarise(path: Path): + data = path.read_bytes() + pdb = RekordboxPdb(is_ext=False, _io=KaitaiStream(BytesIO(data))) + print(f"\n=== {path} ({path.stat().st_size} bytes) ===") + print(f"page_size={pdb.len_page} num_tables={pdb.num_tables} next_unused_page={pdb.next_unused_page} sequence={pdb.sequence}") + for tbl in pdb.tables: + ptype = PAGE_TYPE_NAMES.get(tbl.type.value, str(tbl.type)) + page_count = 0 + row_count = 0 + err = None + try: + ref = tbl.first_page + while True: + page = ref.body + page_count += 1 + if page.is_data_page: + for rg in page.row_groups: + for rpw in rg.rows: + if rpw.present: + row_count += 1 + if ref.index == tbl.last_page.index: + break + ref = page.next_page + if page_count > 5000: + break + except Exception as e: + err = f"{type(e).__name__}: {e}" + line = f" {ptype:24s} first={tbl.first_page.index:>4} last={tbl.last_page.index:>4} pages={page_count:>3} rows={row_count}" + if err: + line += f" ERR={err}" + print(line) + return pdb + + +def main(): + base = Path(__file__).parent + ref = base / "data/MAPLE/PIONEER/rekordbox/export.pdb" + ours = base / "data/usb/PIONEER/rekordbox/export.pdb" + summarise(ref) + summarise(ours) + + +if __name__ == "__main__": + main() diff --git a/maple-py/pyproject.toml b/maple-py/pyproject.toml index 7e659cb..188a9e8 100644 --- a/maple-py/pyproject.toml +++ b/maple-py/pyproject.toml @@ -8,9 +8,14 @@ authors = [ ] requires-python = ">=3.14" dependencies = [ + "kaitaistruct>=0.11", "mutagen>=1.47.0", + "pillow>=12.2.0", ] +[project.scripts] +maple = "maple.cli:main" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/maple-py/src/kaitai/__init__.py b/maple-py/src/kaitai/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/maple-py/src/kaitai/rekordbox_anlz.ksy b/maple-py/src/kaitai/rekordbox_anlz.ksy new file mode 100644 index 0000000..78582b3 --- /dev/null +++ b/maple-py/src/kaitai/rekordbox_anlz.ksy @@ -0,0 +1,673 @@ +meta: + id: rekordbox_anlz + title: rekordbox track analysis file + application: rekordbox + file-extension: + - dat + - ext + license: EPL-1.0 + endian: be + +doc: | + These files are created by rekordbox when analyzing audio tracks + to facilitate DJ performance. They include waveforms, beat grids + (information about the precise time at which each beat occurs), + time indices to allow efficient seeking to specific positions + inside variable bit-rate audio streams, and lists of memory cues + and loop points. They are used by Pioneer professional DJ + equipment. + + The format has been reverse-engineered to facilitate sophisticated + integrations with light and laser shows, videos, and other musical + instruments, by supporting deep knowledge of what is playing and + what is coming next through monitoring the network communications + of the players. + +doc-ref: https://reverseengineering.stackexchange.com/questions/4311/help-reversing-a-edb-database-file-for-pioneers-rekordbox-software + +seq: + - id: magic + contents: "PMAI" + doc: Identifies this as an analysis file. + - id: len_header + type: u4 + doc: | + The number of bytes of this header section. + - id: len_file + type: u4 + doc: | + The number of bytes in the entire file. + - size: len_header - _io.pos + - id: sections + type: tagged_section + repeat: eos + doc: | + The remainder of the file is a sequence of type-tagged sections, + identified by a four-byte magic sequence. + +types: + tagged_section: + doc: | + A type-tagged file section, identified by a four-byte magic + sequence, with a header specifying its length, and whose payload + is determined by the type tag. + seq: + - id: fourcc + type: s4 + enum: section_tags + doc: | + A tag value indicating what kind of section this is. + - id: len_header + type: u4 + doc: | + The size, in bytes, of the header portion of the tag. + - id: len_tag + type: u4 + doc: | + The size, in bytes, of this entire tag, counting the header. + - id: body + size: len_tag - 12 + type: + switch-on: fourcc + cases: + 'section_tags::cues_2': cue_extended_tag # PCO2 + 'section_tags::cues': cue_tag # PCOB + 'section_tags::path': path_tag # PPTH + 'section_tags::beat_grid': beat_grid_tag # PQTZ + 'section_tags::vbr': vbr_tag # PVBR + 'section_tags::wave_preview': wave_preview_tag # PWAV + 'section_tags::wave_tiny': wave_preview_tag # PWV2 + 'section_tags::wave_scroll': wave_scroll_tag # PWV3, seen in .EXT + 'section_tags::wave_color_preview': wave_color_preview_tag # PWV4, in .EXT + 'section_tags::wave_color_scroll': wave_color_scroll_tag # PWV5, in .EXT + 'section_tags::wave_3band_preview': wave_3band_preview_tag # P@V6, in .2EX + 'section_tags::wave_3band_scroll': wave_3band_scroll_tag # PWV7, in .2EX + 'section_tags::song_structure': song_structure_tag # PSSI, in .EXT + _: unknown_tag + -webide-representation: '{fourcc}' + + + beat_grid_tag: + doc: | + Holds a list of all the beats found within the track, recording + their bar position, the time at which they occur, and the tempo + at that point. + seq: + - type: u4 + - type: u4 # @flesniak says this is always 0x80000 + - id: num_beats + type: u4 + doc: | + The number of beat entries which follow. + - id: beats + type: beat_grid_beat + repeat: expr + repeat-expr: num_beats + doc: The entries of the beat grid. + + beat_grid_beat: + doc: | + Describes an individual beat in a beat grid. + seq: + - id: beat_number + type: u2 + doc: | + The position of the beat within its musical bar, where beat 1 + is the down beat. + - id: tempo + type: u2 + doc: | + The tempo at the time of this beat, in beats per minute, + multiplied by 100. + - id: time + type: u4 + doc: | + The time, in milliseconds, at which this beat occurs when + the track is played at normal (100%) pitch. + + cue_tag: + doc: | + Stores either a list of ordinary memory cues and loop points, or + a list of hot cues and loop points. + seq: + - id: type + type: u4 + enum: cue_list_type + doc: | + Identifies whether this tag stores ordinary or hot cues. + - size: 2 + - id: num_cues + type: u2 + doc: | + The length of the cue list. + - id: memory_count + type: u4 + doc: | + Unsure what this means. + - id: cues + type: cue_entry + repeat: expr + repeat-expr: num_cues + + cue_entry: + doc: | + A cue list entry. Can either represent a memory cue or a loop. + seq: + - id: magic + contents: "PCPT" + doc: Identifies this as a cue list entry (cue point). + - id: len_header + type: u4 + - id: len_entry + type: u4 + - id: hot_cue + type: u4 + doc: | + If zero, this is an ordinary memory cue, otherwise this a + hot cue with the specified number. + - id: status + type: u4 + enum: cue_entry_status + doc: | + Indicates if this is an active loop. + - type: u4 # Seems to always be 0x10000 + - id: order_first + type: u2 + doc: | + @flesniak says: "0xffff for first cue, 0,1,3 for next" + - id: order_last + type: u2 + doc: | + @flesniak says: "1,2,3 for first, second, third cue, 0xffff for last" + - id: type + type: u1 + enum: cue_entry_type + doc: | + Indicates whether this is a memory cue or a loop. + - size: 3 # seems to always be 1000 + - id: time + type: u4 + doc: | + The position, in milliseconds, at which the cue point lies + in the track. + - id: loop_time + type: u4 + doc: | + The position, in milliseconds, at which the player loops + back to the cue time if this is a loop. + - size: 16 + + cue_extended_tag: + doc: | + A variation of cue_tag which was introduced with the nxs2 line, + and adds descriptive names. (Still comes in two forms, either + holding memory cues and loop points, or holding hot cues and + loop points.) Also includes hot cues D through H and color assignment. + seq: + - id: type + type: u4 + enum: cue_list_type + doc: | + Identifies whether this tag stores ordinary or hot cues. + - id: num_cues + type: u2 + doc: | + The length of the cue comment list. + - size: 2 + - id: cues + type: cue_extended_entry + repeat: expr + repeat-expr: num_cues + + cue_extended_entry: + doc: | + A cue extended list entry. Can either describe a memory cue or a + loop. + seq: + - id: magic + contents: "PCP2" + doc: Identifies this as an extended cue list entry (cue point). + - id: len_header + type: u4 + - id: len_entry + type: u4 + - id: hot_cue + type: u4 + doc: | + If zero, this is an ordinary memory cue, otherwise this a + hot cue with the specified number. + - id: type + type: u1 + enum: cue_entry_type + doc: | + Indicates whether this is a regular cue point or a loop. + - size: 3 # seems to always be 1000 + - id: time + type: u4 + doc: | + The position, in milliseconds, at which the cue point lies + in the track. + - id: loop_time + type: u4 + doc: | + The position, in milliseconds, at which the player loops + back to the cue time if this is a loop. + - id: color_id + type: u1 + doc: | + References a row in the colors table if this is a memory cue or loop + and has been assigned a color. + - size: 7 + - id: loop_numerator + type: u2 + doc: | + The numerator of the loop length in beats. + Zero if the loop is not quantized. + - id: loop_denominator + type: u2 + doc: | + The denominator of the loop length in beats. + Zero if the loop is not quantized. + - id: len_comment + type: u4 + if: len_entry > 43 + - id: comment + type: str + size: len_comment + encoding: utf-16be + doc: | + The comment assigned to this cue by the DJ, if any, with a trailing NUL. + if: len_entry > 43 + - id: color_code + type: u1 + doc: | + A lookup value for a color table? We use this to index to the hot cue colors shown in rekordbox. + if: (len_entry - len_comment) > 44 + - id: color_red + type: u1 + doc: | + The red component of the hot cue color to be displayed. + if: (len_entry - len_comment) > 45 + - id: color_green + type: u1 + doc: | + The green component of the hot cue color to be displayed. + if: (len_entry - len_comment) > 46 + - id: color_blue + type: u1 + doc: | + The blue component of the hot cue color to be displayed. + if: (len_entry - len_comment) > 47 + - size: len_entry - 48 - len_comment # The remainder after the color + if: (len_entry - len_comment) > 48 + + path_tag: + doc: | + Stores the file path of the audio file to which this analysis + applies. + seq: + - id: len_path + type: u4 + - id: path + type: str + size: len_path - 2 + encoding: utf-16be + if: len_path > 1 + -webide-representation: '{path}' + + vbr_tag: + doc: | + Stores an index allowing rapid seeking to particular times + within a variable-bitrate audio file. + seq: + - type: u4 + - id: index + type: u4 + repeat: expr + repeat-expr: 400 + + wave_preview_tag: + doc: | + Stores a waveform preview image suitable for display above + the touch strip for jumping to a track position. + seq: + - id: len_data + type: u4 + doc: | + The length, in bytes, of the preview data itself. This is + slightly redundant because it can be computed from the + length of the tag. + - type: u4 # This seems to always have the value 0x10000 + - id: data + size: len_data + doc: | + The actual bytes of the waveform preview. + if: _parent.len_tag > _parent.len_header + + wave_scroll_tag: + doc: | + A larger waveform image suitable for scrolling along as a track + plays. + seq: + - id: len_entry_bytes + type: u4 + doc: | + The size of each entry, in bytes. Seems to always be 1. + - id: len_entries + type: u4 + doc: | + The number of waveform data points, each of which takes one + byte. + - type: u4 # Always 0x960000? + - id: entries + size: len_entries * len_entry_bytes + + wave_color_preview_tag: + doc: | + A larger, colorful waveform preview image suitable for display + above the touch strip for jumping to a track position on newer + high-resolution players. + seq: + - id: len_entry_bytes + type: u4 + doc: | + The size of each entry, in bytes. Seems to always be 6. + - id: len_entries + type: u4 + doc: | + The number of waveform data points, each of which takes one + byte for each of six channels of information. + - type: u4 + - id: entries + size: len_entries * len_entry_bytes + + wave_color_scroll_tag: + doc: | + A larger, colorful waveform image suitable for scrolling along + as a track plays on newer high-resolution hardware. Also + contains a higher-resolution blue/white waveform. + seq: + - id: len_entry_bytes + type: u4 + doc: | + The size of each entry, in bytes. Seems to always be 2. + - id: len_entries + type: u4 + doc: | + The number of columns of waveform data (this matches the + non-color waveform length. + - type: u4 + - id: entries + size: len_entries * len_entry_bytes + + wave_3band_preview_tag: + doc: | + The minimalist CDJ-3000 waveform preview image suitable for display + above the touch strip for jumping to a track position. + seq: + - id: len_entry_bytes + type: u4 + doc: | + The size of each entry, in bytes. Seems to always be 3. + - id: len_entries + type: u4 + doc: | + The number of waveform data points, each of which takes one + byte for each of six channels of information. + - id: entries + size: len_entries * len_entry_bytes + + wave_3band_scroll_tag: + doc: | + The minimalist CDJ-3000 waveform image suitable for scrolling along + as a track plays on newer high-resolution hardware. + seq: + - id: len_entry_bytes + type: u4 + doc: | + The size of each entry, in bytes. Seems to always be 3. + - id: len_entries + type: u4 + doc: | + The number of columns of waveform data (this matches the + non-color waveform length. + - type: u4 + - id: entries + size: len_entries * len_entry_bytes + + song_structure_tag: + doc: | + Stores the song structure, also known as phrases (intro, verse, + bridge, chorus, up, down, outro). + seq: + - id: len_entry_bytes + type: u4 + doc: | + The size of each entry, in bytes. Seems to always be 24. + - id: len_entries + type: u2 + doc: | + The number of phrases. + - id: body + type: song_structure_body + doc: | + The rest of the tag, which needs to be unmasked before it + can be parsed. + size-eos: true + process: 'xor(is_masked ? mask : [0])' + instances: + c: + value: len_entries + mask: + value: | + [ + (0xCB+c).as, (0xE1+c).as, (0xEE+c).as, (0xFA+c).as, (0xE5+c).as, (0xEE+c).as, (0xAD+c).as, (0xEE+c).as, + (0xE9+c).as, (0xD2+c).as, (0xE9+c).as, (0xEB+c).as, (0xE1+c).as, (0xE9+c).as, (0xF3+c).as, (0xE8+c).as, + (0xE9+c).as, (0xF4+c).as, (0xE1+c).as + ].as + raw_mood: + type: u2 + pos: 6 + doc: | + This is a way to tell whether the rest of the tag has been masked. The value is supposed + to range from 1 to 3, but in masked files it will be much larger. + is_masked: + value: 'raw_mood > 20' # This is almost certainly not true for an unmasked file. + -webide-representation: '{body.mood}' + + + song_structure_body: + doc: | + Stores the rest of the song structure tag, which can only be + parsed after unmasking. + seq: + - id: mood + type: u2 + enum: track_mood + doc: | + The mood which rekordbox assigns the track as a whole during phrase analysis. + - size: 6 + - id: end_beat + type: u2 + doc: | + The beat number at which the last phrase ends. The track may + continue after the last phrase ends. If this is the case, it will + mostly be silence. + - size: 2 + - id: raw_bank + type: u1 + doc: | + Number identifying a stylistic bank which can be assigned to the track in rekordbox Lighting mode. + - size: 1 + - id: entries + type: song_structure_entry + repeat: expr + repeat-expr: _parent.len_entries + instances: + bank: + value: raw_bank + enum: track_bank + doc: | + The stylistic bank which can be assigned to the track in rekordbox Lighting mode, if raw_bank has a legal value. + if: 'raw_bank < 9' + + + song_structure_entry: + doc: | + A song structure entry, represents a single phrase. + seq: + - id: index + type: u2 + doc: | + The absolute number of the phrase, starting at one. + - id: beat + type: u2 + doc: | + The beat number at which the phrase starts. + - id: kind + type: + switch-on: _parent.mood + cases: + 'track_mood::high': phrase_high + 'track_mood::mid': phrase_mid + 'track_mood::low': phrase_low + _: phrase_mid # We don't recognize this mood, so pick a generic interpretation. + doc: | + The kind of phrase as displayed in rekordbox. + - size: 1 + - id: k1 + type: u1 + doc: One of three flags that identify phrase kind variants in high-mood tracks. + - size: 1 + - id: k2 + type: u1 + doc: One of three flags that identify phrase kind variants in high-mood tracks. + - size: 1 + - id: b + type: u1 + doc: | + Flags how many more beat numbers are in a high-mood "Up 3" phrase. + - id: beat2 + type: u2 + doc: | + Extra beat number (falling within phrase) always present in high-mood "Up 3" phrases. + - id: beat3 + type: u2 + doc: | + Extra beat number (falling within phrase, larger than beat2) + present in high-mood "Up 3" phrases when b has value 1. + - id: beat4 + type: u2 + doc: | + Extra beat number (falling within phrase, larger than beat3) + present in high-mood "Up 3" phrases when b has value 1. + - size: 1 + - id: k3 + type: u1 + doc: One of three flags that identify phrase kind variants in high-mood tracks. + - size: 1 + - id: fill + type: u1 + doc: | + If nonzero, fill-in is present at end of phrase. + - id: beat_fill + type: u2 + doc: | + The beat number at which fill-in starts. + -webide-representation: '{kind.id}' + + phrase_high: + seq: + - id: id + type: u2 + enum: mood_high_phrase + + phrase_mid: + seq: + - id: id + type: u2 + enum: mood_mid_phrase + + phrase_low: + seq: + - id: id + type: u2 + enum: mood_low_phrase + + unknown_tag: {} + +enums: + section_tags: + 0x50434f42: cues # PCOB + 0x50434f32: cues_2 # PCO2 (seen in .EXT) + 0x50505448: path # PPTH + 0x50564252: vbr # PVBR + 0x5051545a: beat_grid # PQTZ + 0x50574156: wave_preview # PWAV + 0x50575632: wave_tiny # PWV2 + 0x50575633: wave_scroll # PWV3 (seen in .EXT) + 0x50575634: wave_color_preview # PWV4 (seen in .EXT) + 0x50575635: wave_color_scroll # PWV5 (seen in .EXT) + 0x50535349: song_structure # PSSI (seen in .EXT) + 0x50575636: wave_3band_preview # PWV6 (seen in .2EX) + 0x50575637: wave_3band_scroll # PWV7 (seen in .2EX) + + cue_list_type: + 0: memory_cues + 1: hot_cues + + cue_entry_type: + 1: memory_cue + 2: loop + + cue_entry_status: + 0: disabled + 1: enabled + 4: active_loop + + track_mood: + 1: high + 2: mid + 3: low + + mood_low_phrase: + 1: intro + 2: verse_1 + 3: verse_1b # Just displayed as "Verse 1" in rekordbox. + 4: verse_1c # Just displayed as "Verse 1" in rekordbox. + 5: verse_2 + 6: verse_2b # Just displayed as "Verse 2" in rekordbox. + 7: verse_2c # Just displayed as "Verse 2" in rekordbox. + 8: bridge + 9: chorus + 10: outro + + mood_mid_phrase: + 1: intro + 2: verse_1 + 3: verse_2 + 4: verse_3 + 5: verse_4 + 6: verse_5 + 7: verse_6 + 8: bridge + 9: chorus + 10: outro + + mood_high_phrase: + 1: intro + 2: up + 3: down + 5: chorus + 6: outro + + track_bank: + 0: default + 1: cool + 2: natural + 3: hot + 4: subtle + 5: warm + 6: vivid + 7: club_1 + 8: club_2 diff --git a/maple-py/src/kaitai/rekordbox_pdb.ksy b/maple-py/src/kaitai/rekordbox_pdb.ksy new file mode 100644 index 0000000..152bdce --- /dev/null +++ b/maple-py/src/kaitai/rekordbox_pdb.ksy @@ -0,0 +1,1171 @@ +meta: + id: rekordbox_pdb + title: DeviceSQL database export (probably generated by rekordbox) + application: rekordbox + file-extension: + - pdb + license: EPL-1.0 + endian: le + +doc: | + This is a relational database format designed to be efficiently used + by very low power devices (there were deployments on 16 bit devices + with 32K of RAM). Today you are most likely to encounter it within + the Pioneer Professional DJ ecosystem, because it is the format that + their rekordbox software uses to write USB and SD media which can be + mounted in DJ controllers and used to play and mix music. + + It has been reverse-engineered to facilitate sophisticated + integrations with light and laser shows, videos, and other musical + instruments, by supporting deep knowledge of what is playing and + what is coming next through monitoring the network communications of + the players. + + The file is divided into fixed-size blocks. The first block has a + header that establishes the block size, and lists the tables + available in the database, identifying their types and the index of + the first of the series of linked pages that make up that table. + + Each table is made up of a series of rows which may be spread across + any number of pages. The pages start with a header describing the + page and linking to the next page. The rest of the page is used as a + heap: rows are scattered around it, and located using an index + structure that builds backwards from the end of the page. Each row + of a given type has a fixed size structure which links to any + variable-sized strings by their offsets within the page. + + As changes are made to the table, some records may become unused, + and there may be gaps within the heap that are too small to be used + by other data. There is a bit map in the row index that identifies + which rows are actually present. Rows that are not present must be + ignored: they do not contain valid (or even necessarily well-formed) + data. + + The majority of the work in reverse-engineering this format was + performed by @henrybetts and @flesniak, for which I am hugely + grateful. @GreyCat helped me learn the intricacies (and best + practices) of Kaitai far faster than I would have managed on my own. + +doc-ref: https://github.com/Deep-Symmetry/crate-digger/blob/master/doc/Analysis.pdf + +params: + - id: is_ext + type: bool + doc: | + Indicates whether the database schema is export or exportExt. + Set this to true when parsing an exportExt.pdb file. + +seq: + - type: u4 + doc: | + Unknown purpose, perhaps an unoriginal signature, seems to + always have the value 0. + - id: len_page + type: u4 + doc: | + The database page size, in bytes. Pages are referred to by + index, so this size is needed to calculate their offset, and + table pages have a row index structure which is built from the + end of the page backwards, so finding that also requires this + value. + - id: num_tables + type: u4 + doc: | + Determines the number of table entries that are present. Each + table is a linked list of pages containing rows of a particular + type. + - id: next_unused_page + type: u4 + doc: | + @flesniak said: "Not used as any `empty_candidate`, points + past the end of the file." + - type: u4 + - id: sequence + type: u4 + doc: | + Sequence number incremented during every edit of the database. + - id: gap + contents: [0, 0, 0, 0] + doc: | + Only exposed until + https://github.com/kaitai-io/kaitai_struct/issues/825 can be + fixed. + - id: tables + type: table + repeat: expr + repeat-expr: num_tables + doc: | + Describes and links to the tables present in the database. + +types: + table: + doc: | + Each table is a linked list of pages containing rows of a single + type. This header describes the nature of the table and links to + its pages by index. + seq: + - id: type + type: u4 + enum: page_type + if: not _root.is_ext + doc: | + Identifies the kind of rows that are found in this table. + - id: type_ext + type: u4 + enum: page_type_ext + if: _root.is_ext + doc: | + Identifies the kind of rows that are found in this table from an exportExt.pdb file. + - id: empty_candidate + type: u4 + - id: first_page + type: page_ref + doc: | + Links to the chain of pages making up that table. The first + page seems to always contain similar garbage patterns and + zero rows, but the next page it links to contains the start + of the meaningful data rows. + - id: last_page + type: page_ref + doc: | + Holds the index of the last page that makes up this table. + When following the linked list of pages of the table, you + either need to stop when you reach this page, or when you + notice that the `next_page` link you followed took you to a + page of a different `type`. + -webide-representation: '{type}' + + page_ref: + doc: | + An index which points to a table page (its offset can be found + by multiplying the index by the `page_len` value in the file + header). This type allows the linked page to be lazy loaded. + seq: + - id: index + type: u4 + doc: | + Identifies the desired page number. + instances: + body: + doc: | + When referenced, loads the specified page and parses its + contents appropriately for the type of data it contains. + io: _root._io + pos: _root.len_page * index + size: _root.len_page + type: page + + page: + doc: | + A table page, consisting of a short header describing the + content of the page and linking to the next page, followed by a + heap in which row data is found. At the end of the page there is + an index which locates all rows present in the heap via their + offsets past the end of the page header. + meta: + bit-endian: le + seq: + - id: gap + contents: [0, 0, 0, 0] + doc: | + Only exposed until + https://github.com/kaitai-io/kaitai_struct/issues/825 can be + fixed. + - id: page_index + doc: Matches the index we used to look up the page, sanity check? + type: u4 + - id: type + type: u4 + enum: page_type + if: not _root.is_ext + doc: | + Identifies the type of information stored in the rows of this page. + - id: type_ext + type: u4 + enum: page_type_ext + if: _root.is_ext + doc: | + Identifies the type of information stored in the rows of this page in an exportExt.pdb file. + - id: next_page + doc: | + Index of the next page containing this type of rows. Points past + the end of the file if there are no more. + type: page_ref + - id: sequence + type: u4 + doc: | + Sequence number updated to the value of sequence from the database + header when this page is edited. The value is copied before the + value of sequence in the database header is incremented, i.e. + the one in the database header is the "next" page sequence number. + - size: 4 + - id: num_row_offsets + type: b13 + doc: | + Seems to hold the number of row offsets that have ever been + allocated, including those that are no longer valid. + - id: num_rows + type: b11 + doc: The number of valid rows currently present in the page. + - id: page_flags + type: u1 + doc: | + @flesniak said: "strange pages: 0x44, 0x64; otherwise seen: 0x24, 0x34" + - id: free_size + type: u2 + doc: | + Unused space (in bytes) in the page heap, excluding the row + index at end of page. + - id: used_size + type: u2 + doc: | + The number of bytes that are in use in the page heap. + - id: transaction_row_count + type: u2 + doc: | + The number of rows touched in the last transaction on this page, + or 0x1fff if the last transaction failed. + - id: transaction_row_index + type: u2 + doc: | + The index of the first row touched in the last transaction on this page, + or 0x1fff if the last transaction failed. + - type: u2 + doc: | + @flesniak said: "1004 for strange blocks, 0 otherwise" + - type: u2 + doc: | + @flesniak said: "always 0 except 1 for history pages, num + entries for strange pages?" + - id: heap + size-eos: true + if: 'false' # never true, but stores pos + instances: + is_data_page: + value: page_flags & 0x40 == 0 + -webide-parse-mode: eager + heap_pos: + value: _io.pos + num_row_groups: + value: '(num_row_offsets - 1) / 16 + 1' + doc: | + The number of row groups that are present in the index. Each + group can hold up to sixteen rows, but `row_present_flags` + must be consulted to determine whether each is valid. + row_groups: + type: 'row_group(_index)' + repeat: expr + repeat-expr: num_row_groups + doc: | + The actual row groups making up the row index. Each group + can hold up to sixteen rows. Non-data pages do not have + actual rows, and attempting to parse them can crash. + if: is_data_page + + row_group: + doc: | + A group of row indices, which are built backwards from the end + of the page. Holds up to sixteen row offsets, along with a bit + mask that indicates whether each row is actually present in the + table. + params: + - id: group_index + type: u2 + doc: | + Identifies which group is being generated. They build backwards + from the end of the page. + instances: + base: + value: '_root.len_page - (group_index * 0x24)' + doc: | + The starting point of this group of row indices. + row_present_flags: + pos: base - 4 + type: u2 + doc: | + Each bit specifies whether a particular row is present. The + low order bit corresponds to the first row in this index, + whose offset immediately precedes these flag bits. The + second bit corresponds to the row whose offset precedes + that, and so on. + -webide-parse-mode: eager + transaction_row_flags: + pos: base + type: u2 + doc: | + Each bit specifies whether a particular row was touched by + the last transaction on this row group, using the same + layout as row_present_flags. + rows: + type: row_ref(_index) + repeat: expr + repeat-expr: 16 + doc: | + The row offsets in this group. + + row_ref: + doc: | + An offset which points to a row in the table, whose actual + presence is controlled by one of the bits in + `row_present_flags`. This instance allows the row itself to be + lazily loaded, unless it is not present, in which case there is + no content to be loaded. + params: + - id: row_index + type: u2 + doc: | + Identifies which row within the row index this reference + came from, so the correct flag can be checked for the row + presence and the correct row offset can be found. + instances: + ofs_row: + pos: '_parent.base - (6 + (2 * row_index))' + type: u2 + doc: | + The offset of the start of the row (in bytes past the end of + the page header). + row_base: + value: ofs_row + _parent._parent.heap_pos + doc: | + The location of this row relative to the start of the page. + A variety of pointers (such as all device_sql_string values) + are calculated with respect to this position. + present: + value: '(((_parent.row_present_flags >> row_index) & 1) != 0 ? true : false)' + doc: | + Indicates whether the row index considers this row to be + present in the table. Will be `false` if the row has been + deleted. + -webide-parse-mode: eager + body: + pos: row_base + if: present and not _root.is_ext + type: + switch-on: _parent._parent.type + cases: + 'page_type::albums': album_row + 'page_type::artists': artist_row + 'page_type::artwork': artwork_row + 'page_type::colors': color_row + 'page_type::genres': genre_row + 'page_type::keys': key_row + 'page_type::labels': label_row + 'page_type::playlist_tree': playlist_tree_row + 'page_type::playlist_entries': playlist_entry_row + 'page_type::history_playlists': history_playlist_row + 'page_type::history_entries': history_entry_row + 'page_type::tracks': track_row + doc: | + The actual content of the row, as long as it is present. + -webide-parse-mode: eager + body_ext: + pos: row_base + if: present and _root.is_ext + type: + switch-on: _parent._parent.type_ext + cases: + 'page_type_ext::tags': tag_row + 'page_type_ext::tag_tracks': tag_track_row + doc: | + The actual content of the row in an exportExt.pdb file, as long as it is present. + -webide-parse-mode: eager + -webide-representation: '{body.name.body.text}{body.title.body.text}{body_ext.name.body.text} ({body.id}{body_ext.id})' + + album_row: + doc: | + A row that holds an album name and ID. + seq: + - id: subtype + type: u2 + doc: | + Usually 0x80, but 0x84 means we have a long name offset + embedded in the row. + - id: index_shift + type: u2 + doc: TODO name from @flesniak, but what does it mean? + - type: u4 + - id: artist_id + type: u4 + doc: | + Identifies the artist associated with the album. + - id: id + type: u4 + doc: | + The unique identifier by which this album can be requested + and linked from other rows (such as tracks). + - type: u4 + - type: u1 + doc: | + @flesniak says: "always 0x03, maybe an unindexed empty string" + - id: ofs_name_near + type: u1 + doc: | + The location of the variable-length name string, relative to + the start of this row, unless subtype is 0x84. + instances: + ofs_name_far: + pos: _parent.row_base + 0x16 + type: u2 + doc: | + For names that might be further than 0xff bytes from the + start of this row, this holds a two-byte offset, and is + signalled by the subtype value. + if: subtype & 0x04 == 0x04 + name: + pos: '_parent.row_base + (subtype & 0x04 == 0x04? ofs_name_far : ofs_name_near)' + type: device_sql_string + doc: | + The name of this album. + -webide-parse-mode: eager + + artist_row: + doc: | + A row that holds an artist name and ID. + seq: + - id: subtype + type: u2 + doc: | + Usually 0x60, but 0x64 means we have a long name offset + embedded in the row. + - id: index_shift + type: u2 + doc: TODO name from @flesniak, but what does it mean? + - id: id + type: u4 + doc: | + The unique identifier by which this artist can be requested + and linked from other rows (such as tracks). + - type: u1 + doc: | + @flesniak says: "always 0x03, maybe an unindexed empty string" + - id: ofs_name_near + type: u1 + doc: | + The location of the variable-length name string, relative to + the start of this row, unless subtype is 0x64. + instances: + ofs_name_far: + pos: _parent.row_base + 0x0a + type: u2 + doc: | + For names that might be further than 0xff bytes from the + start of this row, this holds a two-byte offset, and is + signalled by the subtype value. + if: subtype & 0x04 == 0x04 + name: + pos: '_parent.row_base + (subtype & 0x04 == 0x04? ofs_name_far : ofs_name_near)' + type: device_sql_string + doc: | + The name of this artist. + -webide-parse-mode: eager + + artwork_row: + doc: | + A row that holds the path to an album art image file and the + associated artwork ID. + seq: + - id: id + type: u4 + doc: | + The unique identifier by which this art can be requested + and linked from other rows (such as tracks). + - id: path + type: device_sql_string + doc: | + The variable-length file path string at which the art file + can be found. + -webide-representation: '{path.body.text}' + + color_row: + doc: | + A row that holds a color name and the associated ID. + seq: + - size: 5 + - id: id + type: u2 + doc: | + The unique identifier by which this color can be requested + and linked from other rows (such as tracks). + - type: u1 + - id: name + type: device_sql_string + doc: | + The variable-length string naming the color. + + genre_row: + doc: | + A row that holds a genre name and the associated ID. + seq: + - id: id + type: u4 + doc: | + The unique identifier by which this genre can be requested + and linked from other rows (such as tracks). + - id: name + type: device_sql_string + doc: | + The variable-length string naming the genre. + + key_row: + doc: | + A row that holds a musical key and the associated ID. + seq: + - id: id + type: u4 + doc: | + The unique identifier by which this key can be requested + and linked from other rows (such as tracks). + - id: id2 + type: u4 + doc: | + Seems to be a second copy of the ID? + - id: name + type: device_sql_string + doc: | + The variable-length string naming the key. + + label_row: + doc: | + A row that holds a label name and the associated ID. + seq: + - id: id + type: u4 + doc: | + The unique identifier by which this label can be requested + and linked from other rows (such as tracks). + - id: name + type: device_sql_string + doc: | + The variable-length string naming the label. + + playlist_tree_row: + doc: | + A row that holds a playlist name, ID, indication of whether it + is an ordinary playlist or a folder of other playlists, a link + to its parent folder, and its sort order. + seq: + - id: parent_id + type: u4 + doc: | + The ID of the `playlist_tree_row` in which this one can be + found, or `0` if this playlist exists at the root level. + - size: 4 + - id: sort_order + type: u4 + doc: | + The order in which the entries of this playlist are sorted. + - id: id + type: u4 + doc: | + The unique identifier by which this playlist or folder can + be requested and linked from other rows. + - id: raw_is_folder + type: u4 + doc: | + Has a non-zero value if this is actually a folder rather + than a playlist. + - id: name + type: device_sql_string + doc: | + The variable-length string naming the playlist. + instances: + is_folder: + value: raw_is_folder != 0 + -webide-parse-mode: eager + + playlist_entry_row: + doc: | + A row that associates a track with a position in a playlist. + seq: + - id: entry_index + type: u4 + doc: | + The position within the playlist represented by this entry. + - id: track_id + type: u4 + doc: | + The track found at this position in the playlist. + - id: playlist_id + type: u4 + doc: | + The playlist to which this entry belongs. + + history_playlist_row: + doc: | + A row that holds a history playlist ID and name, linking to + the track IDs captured during a performance on the player. + seq: + - id: id + type: u4 + doc: | + The unique identifier by which this history playlist can + be requested. + - id: name + type: device_sql_string + doc: | + The variable-length string naming the playlist. + + history_entry_row: + doc: | + A row that associates a track with a position in a history playlist. + seq: + - id: track_id + type: u4 + doc: | + The track found at this position in the playlist. + - id: playlist_id + type: u4 + doc: | + The history playlist to which this entry belongs. + - id: entry_index + type: u4 + doc: | + The position within the playlist represented by this entry. + + track_row: + doc: | + A row that describes a track that can be played, with many + details about the music, and links to other tables like artists, + albums, keys, etc. + seq: + - id: subtype + type: u2 + doc: | + Seems to always be 0x24, and bit 0x04 being set means it uses sixteen-bit offsets, + as it does in other tables. Track rows are always big enough to need that size offsets. + - id: index_shift + type: u2 + doc: TODO name from @flesniak, but what does it mean? + - id: bitmask + type: u4 + doc: TODO what do the bits mean? + - id: sample_rate + type: u4 + doc: | + Playback sample rate of the audio file. + - id: composer_id + type: u4 + doc: | + References a row in the artist table if the composer is + known. + - id: file_size + type: u4 + doc: | + The length of the audio file, in bytes. + - type: u4 + doc: | + Some ID? Purpose as yet unknown. + - type: u2 + doc: | + From @flesniak: "always 19048?" + - type: u2 + doc: | + From @flesniak: "always 30967?" + - id: artwork_id + type: u4 + doc: | + References a row in the artwork table if there is album art. + - id: key_id + type: u4 + doc: | + References a row in the keys table if the track has a known + main musical key. + - id: original_artist_id + type: u4 + doc: | + References a row in the artwork table if this is a cover + performance and the original artist is known. + - id: label_id + type: u4 + doc: | + References a row in the labels table if the track has a + known record label. + - id: remixer_id + type: u4 + doc: | + References a row in the artists table if the track has a + known remixer. + - id: bitrate + type: u4 + doc: | + Playback bit rate of the audio file. + - id: track_number + type: u4 + doc: | + The position of the track within an album. + - id: tempo + type: u4 + doc: | + The tempo at the start of the track in beats per minute, + multiplied by 100. + - id: genre_id + type: u4 + doc: | + References a row in the genres table if the track has a + known musical genre. + - id: album_id + type: u4 + doc: | + References a row in the albums table if the track has a + known album. + - id: artist_id + type: u4 + doc: | + References a row in the artists table if the track has a + known performer. + - id: id + type: u4 + doc: | + The id by which this track can be looked up; players will + report this value in their status packets when they are + playing the track. + - id: disc_number + type: u2 + doc: | + The number of the disc on which this track is found, if it + is known to be part of a multi-disc album. + - id: play_count + type: u2 + doc: | + The number of times this track has been played. + - id: year + type: u2 + doc: | + The year in which this track was released. + - id: sample_depth + type: u2 + doc: | + The number of bits per sample of the audio file. + - id: duration + type: u2 + doc: | + The length, in seconds, of the track when played at normal + speed. + - type: u2 + doc: | + From @flesniak: "always 41?" + - id: color_id + type: u1 + doc: | + References a row in the colors table if the track has been + assigned a color. + - id: rating + type: u1 + doc: | + The number of stars to display for the track, 0 to 5. + - type: u2 + doc: | + From @flesniak: "always 1?" + - type: u2 + doc: | + From @flesniak: "alternating 2 or 3" + - id: ofs_strings + type: u2 + repeat: expr + repeat-expr: 21 + doc: | + The location, relative to the start of this row, of a + variety of variable-length strings. + instances: + isrc: + type: device_sql_string + pos: _parent.row_base + ofs_strings[0] + doc: | + International Standard Recording Code of track + when known (in mangled format). + -webide-parse-mode: eager + texter: + type: device_sql_string + pos: _parent.row_base + ofs_strings[1] + doc: | + A string of unknown purpose, which @flesniak named. + -webide-parse-mode: eager + unknown_string_2: + type: device_sql_string + pos: _parent.row_base + ofs_strings[2] + doc: | + A string of unknown purpose; @flesniak said "thought + track number -> wrong!" + unknown_string_3: + type: device_sql_string + pos: _parent.row_base + ofs_strings[3] + doc: | + A string of unknown purpose; @flesniak said "strange + strings, often zero length, sometimes low binary values + 0x01/0x02 as content" + unknown_string_4: + type: device_sql_string + pos: _parent.row_base + ofs_strings[4] + doc: | + A string of unknown purpose; @flesniak said "strange + strings, often zero length, sometimes low binary values + 0x01/0x02 as content" + -webide-parse-mode: eager + message: + type: device_sql_string + pos: _parent.row_base + ofs_strings[5] + doc: | + A string of unknown purpose, which @flesniak named. + -webide-parse-mode: eager + kuvo_public: + type: device_sql_string + pos: _parent.row_base + ofs_strings[6] + doc: | + A string whose value is always either empty or "ON", and + which apparently for some insane reason is used, rather than + a single bit somewhere, to control whether the track + information is visible on Kuvo. + -webide-parse-mode: eager + autoload_hot_cues: + type: device_sql_string + pos: _parent.row_base + ofs_strings[7] + doc: | + A string whose value is always either empty or "ON", and + which apparently for some insane reason is used, rather than + a single bit somewhere, to control whether hot-cues are + auto-loaded for the track. + -webide-parse-mode: eager + unknown_string_5: + type: device_sql_string + pos: _parent.row_base + ofs_strings[8] + doc: | + A string of unknown purpose. + -webide-parse-mode: eager + unknown_string_6: + type: device_sql_string + pos: _parent.row_base + ofs_strings[9] + doc: | + A string of unknown purpose, usually empty. + -webide-parse-mode: eager + date_added: + type: device_sql_string + pos: _parent.row_base + ofs_strings[10] + doc: | + A string containing the date this track was added to the collection. + -webide-parse-mode: eager + release_date: + type: device_sql_string + pos: _parent.row_base + ofs_strings[11] + doc: | + A string containing the date this track was released, if known. + -webide-parse-mode: eager + mix_name: + type: device_sql_string + pos: _parent.row_base + ofs_strings[12] + doc: | + A string naming the remix of the track, if known. + -webide-parse-mode: eager + unknown_string_7: + type: device_sql_string + pos: _parent.row_base + ofs_strings[13] + doc: | + A string of unknown purpose, usually empty. + -webide-parse-mode: eager + analyze_path: + type: device_sql_string + pos: _parent.row_base + ofs_strings[14] + doc: | + The file path of the track analysis, which allows rapid + seeking to particular times in variable bit-rate files, + jumping to particular beats, visual waveform previews, and + stores cue points and loops. + -webide-parse-mode: eager + analyze_date: + type: device_sql_string + pos: _parent.row_base + ofs_strings[15] + doc: | + A string containing the date this track was analyzed by rekordbox. + -webide-parse-mode: eager + comment: + type: device_sql_string + pos: _parent.row_base + ofs_strings[16] + doc: | + The comment assigned to the track by the DJ, if any. + -webide-parse-mode: eager + title: + type: device_sql_string + pos: _parent.row_base + ofs_strings[17] + doc: | + The title of the track. + -webide-parse-mode: eager + unknown_string_8: + type: device_sql_string + pos: _parent.row_base + ofs_strings[18] + doc: | + A string of unknown purpose, usually empty. + -webide-parse-mode: eager + filename: + type: device_sql_string + pos: _parent.row_base + ofs_strings[19] + doc: | + The file name of the track audio file. + -webide-parse-mode: eager + file_path: + type: device_sql_string + pos: _parent.row_base + ofs_strings[20] + doc: | + The file path of the track audio file. + -webide-parse-mode: eager + + tag_row: + doc: | + A row that holds a tag name and its ID (found only in exportExt.pdb files). + seq: + - id: subtype + type: u2 + doc: | + Usually 0x0680, but 0x0684 means we have a long name offset + embedded in the row. + - id: tag_index + type: u2 + doc: | + Increasing index for each row in multiples of 0x20. + - type: u8 + doc: | + Seems to always be zero. + - id: category + type: u4 + doc: | + The ID of the tag category this tag belongs to. + If this row represents a tag category, this field is zero. + - id: category_pos + type: u4 + doc: | + The zero-based position of this tag in its category. + If this row represents a tag category, the zero-based position of the category itself in the category list. + - id: id + type: u4 + doc: | + The ID of this tag or tag category. + Referenced by tag_track_row if this row is a tag. + - id: raw_is_category + type: u4 + doc: | + Non-zero when this row stores a tag category instead of a tag. + - type: u1 + doc: | + @flesniak says: "always 0x03, maybe an unindexed empty string" + - id: ofs_name_near + type: u1 + doc: | + The location of the variable-length name string, relative to + the start of this row, unless subtype is 0x64. + - id: ofs_unknown_near + type: u1 + doc: | + Offset to a string that seems always to be empty. + instances: + is_category: + value: raw_is_category != 0 + doc: | + Indicates whether this row stores a tag category instead of a tag. + -webide-parse-mode: eager + ofs_name_far: + pos: _parent.row_base + 0x1e + type: u2 + doc: | + For names that might be further than 0xff bytes from the + start of this row, this holds a two-byte offset, and is + signalled by the subtype value. + if: subtype & 0x04 == 0x04 + name: + pos: '_parent.row_base + (subtype & 0x04 == 0x04? ofs_name_far : ofs_name_near)' + type: device_sql_string + doc: | + The name of this tag or tag category. + -webide-parse-mode: eager + + tag_track_row: + doc: | + A row that associates a track and a tag (found only in exportExt.pdb files). + seq: + - type: u4 + doc: | + Seems to always be zero. + - id: track_id + type: u4 + doc: | + The ID of the track that has a tag assigned to it. + - id: tag_id + type: u4 + doc: | + The ID of the tag that has been assigned to a track. + - type: u4 + doc: | + Seems to always be 0x03 0x00 0x00 0x00. + + device_sql_string: + doc: | + A variable length string which can be stored in a variety of + different encodings. + seq: + - id: length_and_kind + type: u1 + doc: | + Mangled length of an ordinary ASCII string if odd, or a flag + indicating another encoding with a longer length value to + follow. + - id: body + type: + switch-on: length_and_kind + cases: + 0x40: device_sql_long_ascii + 0x90: device_sql_long_utf16le + _: device_sql_short_ascii(length_and_kind) + -webide-parse-mode: eager + -webide-representation: '{body.text}' + + device_sql_short_ascii: + doc: | + An ASCII-encoded string up to 127 bytes long. + params: + - id: length_and_kind + type: u1 + doc: | + Contains the actual length, incremented, doubled, and + incremented again. Go figure. + seq: + - id: text + type: str + size: length - 1 + encoding: ASCII + doc: | + The content of the string. + instances: + length: + value: '(length_and_kind >> 1)' + doc: | + the length extracted of the entire device_sql_short_ascii type + -webide-parse-mode: eager + + device_sql_long_ascii: + doc: | + An ASCII-encoded string preceded by a two-byte length field in a four-byte header. + seq: + - id: length + type: u2 + doc: | + Contains the length of the string in bytes. + - type: u1 + - id: text + type: str + size: length - 4 + encoding: ASCII + doc: | + The content of the string. + + device_sql_long_utf16le: + doc: | + A UTF-16LE-encoded string preceded by a two-byte length field in a four-byte header. + seq: + - id: length + type: u2 + doc: | + Contains the length of the string in bytes, plus four trailing bytes that must be ignored. + - type: u1 + - id: text + type: str + size: length - 4 + encoding: UTF-16LE + doc: | + The content of the string. + +enums: + page_type: + 0: + id: tracks + doc: | + Holds rows describing tracks, such as their title, artist, + genre, artwork ID, playing time, etc. + 1: + id: genres + doc: | + Holds rows naming musical genres, for reference by tracks and searching. + 2: + id: artists + doc: | + Holds rows naming artists, for reference by tracks and searching. + 3: + id: albums + doc: | + Holds rows naming albums, for reference by tracks and searching. + 4: + id: labels + doc: | + Holds rows naming music labels, for reference by tracks and searching. + 5: + id: keys + doc: | + Holds rows naming musical keys, for reference by tracks and searching. + 6: + id: colors + doc: | + Holds rows naming color labels, for reference by tracks and searching. + 7: + id: playlist_tree + doc: | + Holds rows that describe the hierarchical tree structure of + available playlists and folders grouping them. + 8: + id: playlist_entries + doc: | + Holds rows that enumerate the tracks found in playlists and + the playlists they belong to. + 9: + id: unknown_9 + 10: + id: unknown_10 + 11: + id: history_playlists + doc: | + Holds rows that assign IDs and give names to the history playlists + that have been captured by the player, such as "HISTORY 001". + 12: + id: history_entries + doc: | + Holds rows that enumerate the tracks found in history playlists + and the playlists they belong to. + 13: + id: artwork + doc: | + Holds rows pointing to album artwork images. + 14: + id: unknown_14 + 15: + id: unknown_15 + 16: + id: columns + doc: | + TODO figure out and explain + 17: + id: unknown_17 + 18: + id: unknown_18 + 19: + id: history + doc: | + Holds information to help rekordbox sync history playlists. + + page_type_ext: + 0: + id: unknown_0 + 1: + id: unknown_1 + 2: + id: unknown_2 + 3: + id: tags + doc: | + Holds rows naming tags. + 4: + id: tag_tracks + doc: | + Holds rows associating tags and tracks. + 5: + id: unknown_5 + 6: + id: unknown_6 + 7: + id: unknown_7 + 8: + id: unknown_8 diff --git a/maple-py/src/kaitai/rekordbox_pdb.py b/maple-py/src/kaitai/rekordbox_pdb.py new file mode 100644 index 0000000..30913e2 --- /dev/null +++ b/maple-py/src/kaitai/rekordbox_pdb.py @@ -0,0 +1,1553 @@ +# This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild +# type: ignore + +import kaitaistruct +from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from enum import IntEnum + + +if getattr(kaitaistruct, 'API_VERSION', (0, 9)) < (0, 11): + raise Exception("Incompatible Kaitai Struct Python API: 0.11 or later is required, but you have %s" % (kaitaistruct.__version__)) + +class RekordboxPdb(KaitaiStruct): + """This is a relational database format designed to be efficiently used + by very low power devices (there were deployments on 16 bit devices + with 32K of RAM). Today you are most likely to encounter it within + the Pioneer Professional DJ ecosystem, because it is the format that + their rekordbox software uses to write USB and SD media which can be + mounted in DJ controllers and used to play and mix music. + + It has been reverse-engineered to facilitate sophisticated + integrations with light and laser shows, videos, and other musical + instruments, by supporting deep knowledge of what is playing and + what is coming next through monitoring the network communications of + the players. + + The file is divided into fixed-size blocks. The first block has a + header that establishes the block size, and lists the tables + available in the database, identifying their types and the index of + the first of the series of linked pages that make up that table. + + Each table is made up of a series of rows which may be spread across + any number of pages. The pages start with a header describing the + page and linking to the next page. The rest of the page is used as a + heap: rows are scattered around it, and located using an index + structure that builds backwards from the end of the page. Each row + of a given type has a fixed size structure which links to any + variable-sized strings by their offsets within the page. + + As changes are made to the table, some records may become unused, + and there may be gaps within the heap that are too small to be used + by other data. There is a bit map in the row index that identifies + which rows are actually present. Rows that are not present must be + ignored: they do not contain valid (or even necessarily well-formed) + data. + + The majority of the work in reverse-engineering this format was + performed by @henrybetts and @flesniak, for which I am hugely + grateful. @GreyCat helped me learn the intricacies (and best + practices) of Kaitai far faster than I would have managed on my own. + + .. seealso:: + Source - https://github.com/Deep-Symmetry/crate-digger/blob/master/doc/Analysis.pdf + """ + + class PageType(IntEnum): + tracks = 0 + genres = 1 + artists = 2 + albums = 3 + labels = 4 + keys = 5 + colors = 6 + playlist_tree = 7 + playlist_entries = 8 + unknown_9 = 9 + unknown_10 = 10 + history_playlists = 11 + history_entries = 12 + artwork = 13 + unknown_14 = 14 + unknown_15 = 15 + columns = 16 + unknown_17 = 17 + unknown_18 = 18 + history = 19 + + class PageTypeExt(IntEnum): + unknown_0 = 0 + unknown_1 = 1 + unknown_2 = 2 + tags = 3 + tag_tracks = 4 + unknown_5 = 5 + unknown_6 = 6 + unknown_7 = 7 + unknown_8 = 8 + def __init__(self, is_ext, _io, _parent=None, _root=None): + super(RekordboxPdb, self).__init__(_io) + self._parent = _parent + self._root = _root or self + self.is_ext = is_ext + self._read() + + def _read(self): + self._unnamed0 = self._io.read_u4le() + self.len_page = self._io.read_u4le() + self.num_tables = self._io.read_u4le() + self.next_unused_page = self._io.read_u4le() + self._unnamed4 = self._io.read_u4le() + self.sequence = self._io.read_u4le() + self.gap = self._io.read_bytes(4) + if not self.gap == b"\x00\x00\x00\x00": + raise kaitaistruct.ValidationNotEqualError(b"\x00\x00\x00\x00", self.gap, self._io, u"/seq/6") + self.tables = [] + for i in range(self.num_tables): + self.tables.append(RekordboxPdb.Table(self._io, self, self._root)) + + + + def _fetch_instances(self): + pass + for i in range(len(self.tables)): + pass + self.tables[i]._fetch_instances() + + + class AlbumRow(KaitaiStruct): + """A row that holds an album name and ID. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.AlbumRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.subtype = self._io.read_u2le() + self.index_shift = self._io.read_u2le() + self._unnamed2 = self._io.read_u4le() + self.artist_id = self._io.read_u4le() + self.id = self._io.read_u4le() + self._unnamed5 = self._io.read_u4le() + self._unnamed6 = self._io.read_u1() + self.ofs_name_near = self._io.read_u1() + + + def _fetch_instances(self): + pass + _ = self.name + if hasattr(self, '_m_name'): + pass + self._m_name._fetch_instances() + + _ = self.ofs_name_far + if hasattr(self, '_m_ofs_name_far'): + pass + + + @property + def name(self): + """The name of this album. + """ + if hasattr(self, '_m_name'): + return self._m_name + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + (self.ofs_name_far if self.subtype & 4 == 4 else self.ofs_name_near)) + self._m_name = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_name', None) + + @property + def ofs_name_far(self): + """For names that might be further than 0xff bytes from the + start of this row, this holds a two-byte offset, and is + signalled by the subtype value. + """ + if hasattr(self, '_m_ofs_name_far'): + return self._m_ofs_name_far + + if self.subtype & 4 == 4: + pass + _pos = self._io.pos() + self._io.seek(self._parent.row_base + 22) + self._m_ofs_name_far = self._io.read_u2le() + self._io.seek(_pos) + + return getattr(self, '_m_ofs_name_far', None) + + + class ArtistRow(KaitaiStruct): + """A row that holds an artist name and ID. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.ArtistRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.subtype = self._io.read_u2le() + self.index_shift = self._io.read_u2le() + self.id = self._io.read_u4le() + self._unnamed3 = self._io.read_u1() + self.ofs_name_near = self._io.read_u1() + + + def _fetch_instances(self): + pass + _ = self.name + if hasattr(self, '_m_name'): + pass + self._m_name._fetch_instances() + + _ = self.ofs_name_far + if hasattr(self, '_m_ofs_name_far'): + pass + + + @property + def name(self): + """The name of this artist. + """ + if hasattr(self, '_m_name'): + return self._m_name + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + (self.ofs_name_far if self.subtype & 4 == 4 else self.ofs_name_near)) + self._m_name = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_name', None) + + @property + def ofs_name_far(self): + """For names that might be further than 0xff bytes from the + start of this row, this holds a two-byte offset, and is + signalled by the subtype value. + """ + if hasattr(self, '_m_ofs_name_far'): + return self._m_ofs_name_far + + if self.subtype & 4 == 4: + pass + _pos = self._io.pos() + self._io.seek(self._parent.row_base + 10) + self._m_ofs_name_far = self._io.read_u2le() + self._io.seek(_pos) + + return getattr(self, '_m_ofs_name_far', None) + + + class ArtworkRow(KaitaiStruct): + """A row that holds the path to an album art image file and the + associated artwork ID. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.ArtworkRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.id = self._io.read_u4le() + self.path = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + + + def _fetch_instances(self): + pass + self.path._fetch_instances() + + + class ColorRow(KaitaiStruct): + """A row that holds a color name and the associated ID. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.ColorRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self._unnamed0 = self._io.read_bytes(5) + self.id = self._io.read_u2le() + self._unnamed2 = self._io.read_u1() + self.name = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + + + def _fetch_instances(self): + pass + self.name._fetch_instances() + + + class DeviceSqlLongAscii(KaitaiStruct): + """An ASCII-encoded string preceded by a two-byte length field in a four-byte header. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.DeviceSqlLongAscii, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.length = self._io.read_u2le() + self._unnamed1 = self._io.read_u1() + self.text = (self._io.read_bytes(self.length - 4)).decode(u"ASCII") + + + def _fetch_instances(self): + pass + + + class DeviceSqlLongUtf16le(KaitaiStruct): + """A UTF-16LE-encoded string preceded by a two-byte length field in a four-byte header. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.DeviceSqlLongUtf16le, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.length = self._io.read_u2le() + self._unnamed1 = self._io.read_u1() + self.text = (self._io.read_bytes(self.length - 4)).decode(u"UTF-16LE") + + + def _fetch_instances(self): + pass + + + class DeviceSqlShortAscii(KaitaiStruct): + """An ASCII-encoded string up to 127 bytes long. + """ + def __init__(self, length_and_kind, _io, _parent=None, _root=None): + super(RekordboxPdb.DeviceSqlShortAscii, self).__init__(_io) + self._parent = _parent + self._root = _root + self.length_and_kind = length_and_kind + self._read() + + def _read(self): + self.text = (self._io.read_bytes(self.length - 1)).decode(u"ASCII") + + + def _fetch_instances(self): + pass + + @property + def length(self): + """the length extracted of the entire device_sql_short_ascii type + """ + if hasattr(self, '_m_length'): + return self._m_length + + self._m_length = self.length_and_kind >> 1 + return getattr(self, '_m_length', None) + + + class DeviceSqlString(KaitaiStruct): + """A variable length string which can be stored in a variety of + different encodings. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.DeviceSqlString, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.length_and_kind = self._io.read_u1() + _on = self.length_and_kind + if _on == 144: + pass + self.body = RekordboxPdb.DeviceSqlLongUtf16le(self._io, self, self._root) + elif _on == 64: + pass + self.body = RekordboxPdb.DeviceSqlLongAscii(self._io, self, self._root) + else: + pass + self.body = RekordboxPdb.DeviceSqlShortAscii(self.length_and_kind, self._io, self, self._root) + + + def _fetch_instances(self): + pass + _on = self.length_and_kind + if _on == 144: + pass + self.body._fetch_instances() + elif _on == 64: + pass + self.body._fetch_instances() + else: + pass + self.body._fetch_instances() + + + class GenreRow(KaitaiStruct): + """A row that holds a genre name and the associated ID. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.GenreRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.id = self._io.read_u4le() + self.name = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + + + def _fetch_instances(self): + pass + self.name._fetch_instances() + + + class HistoryEntryRow(KaitaiStruct): + """A row that associates a track with a position in a history playlist. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.HistoryEntryRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.track_id = self._io.read_u4le() + self.playlist_id = self._io.read_u4le() + self.entry_index = self._io.read_u4le() + + + def _fetch_instances(self): + pass + + + class HistoryPlaylistRow(KaitaiStruct): + """A row that holds a history playlist ID and name, linking to + the track IDs captured during a performance on the player. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.HistoryPlaylistRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.id = self._io.read_u4le() + self.name = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + + + def _fetch_instances(self): + pass + self.name._fetch_instances() + + + class KeyRow(KaitaiStruct): + """A row that holds a musical key and the associated ID. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.KeyRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.id = self._io.read_u4le() + self.id2 = self._io.read_u4le() + self.name = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + + + def _fetch_instances(self): + pass + self.name._fetch_instances() + + + class LabelRow(KaitaiStruct): + """A row that holds a label name and the associated ID. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.LabelRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.id = self._io.read_u4le() + self.name = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + + + def _fetch_instances(self): + pass + self.name._fetch_instances() + + + class Page(KaitaiStruct): + """A table page, consisting of a short header describing the + content of the page and linking to the next page, followed by a + heap in which row data is found. At the end of the page there is + an index which locates all rows present in the heap via their + offsets past the end of the page header. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.Page, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.gap = self._io.read_bytes(4) + if not self.gap == b"\x00\x00\x00\x00": + raise kaitaistruct.ValidationNotEqualError(b"\x00\x00\x00\x00", self.gap, self._io, u"/types/page/seq/0") + self.page_index = self._io.read_u4le() + if (not (self._root.is_ext)): + pass + self.type = KaitaiStream.resolve_enum(RekordboxPdb.PageType, self._io.read_u4le()) + + if self._root.is_ext: + pass + self.type_ext = KaitaiStream.resolve_enum(RekordboxPdb.PageTypeExt, self._io.read_u4le()) + + self.next_page = RekordboxPdb.PageRef(self._io, self, self._root) + self.sequence = self._io.read_u4le() + self._unnamed6 = self._io.read_bytes(4) + self.num_row_offsets = self._io.read_bits_int_le(13) + self.num_rows = self._io.read_bits_int_le(11) + self.page_flags = self._io.read_u1() + self.free_size = self._io.read_u2le() + self.used_size = self._io.read_u2le() + self.transaction_row_count = self._io.read_u2le() + self.transaction_row_index = self._io.read_u2le() + self._unnamed14 = self._io.read_u2le() + self._unnamed15 = self._io.read_u2le() + if False: + pass + self.heap = self._io.read_bytes_full() + + + + def _fetch_instances(self): + pass + if (not (self._root.is_ext)): + pass + + if self._root.is_ext: + pass + + self.next_page._fetch_instances() + if False: + pass + + _ = self.row_groups + if hasattr(self, '_m_row_groups'): + pass + for i in range(len(self._m_row_groups)): + pass + self._m_row_groups[i]._fetch_instances() + + + + @property + def heap_pos(self): + if hasattr(self, '_m_heap_pos'): + return self._m_heap_pos + + self._m_heap_pos = self._io.pos() + return getattr(self, '_m_heap_pos', None) + + @property + def is_data_page(self): + if hasattr(self, '_m_is_data_page'): + return self._m_is_data_page + + self._m_is_data_page = self.page_flags & 64 == 0 + return getattr(self, '_m_is_data_page', None) + + @property + def num_row_groups(self): + """The number of row groups that are present in the index. Each + group can hold up to sixteen rows, but `row_present_flags` + must be consulted to determine whether each is valid. + """ + if hasattr(self, '_m_num_row_groups'): + return self._m_num_row_groups + + self._m_num_row_groups = (self.num_row_offsets - 1) // 16 + 1 + return getattr(self, '_m_num_row_groups', None) + + @property + def row_groups(self): + """The actual row groups making up the row index. Each group + can hold up to sixteen rows. Non-data pages do not have + actual rows, and attempting to parse them can crash. + """ + if hasattr(self, '_m_row_groups'): + return self._m_row_groups + + if self.is_data_page: + pass + self._m_row_groups = [] + for i in range(self.num_row_groups): + self._m_row_groups.append(RekordboxPdb.RowGroup(i, self._io, self, self._root)) + + + return getattr(self, '_m_row_groups', None) + + + class PageRef(KaitaiStruct): + """An index which points to a table page (its offset can be found + by multiplying the index by the `page_len` value in the file + header). This type allows the linked page to be lazy loaded. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.PageRef, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.index = self._io.read_u4le() + + + def _fetch_instances(self): + pass + _ = self.body + if hasattr(self, '_m_body'): + pass + self._m_body._fetch_instances() + + + @property + def body(self): + """When referenced, loads the specified page and parses its + contents appropriately for the type of data it contains. + """ + if hasattr(self, '_m_body'): + return self._m_body + + io = self._root._io + _pos = io.pos() + io.seek(self._root.len_page * self.index) + self._raw__m_body = io.read_bytes(self._root.len_page) + _io__raw__m_body = KaitaiStream(BytesIO(self._raw__m_body)) + self._m_body = RekordboxPdb.Page(_io__raw__m_body, self, self._root) + io.seek(_pos) + return getattr(self, '_m_body', None) + + + class PlaylistEntryRow(KaitaiStruct): + """A row that associates a track with a position in a playlist. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.PlaylistEntryRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.entry_index = self._io.read_u4le() + self.track_id = self._io.read_u4le() + self.playlist_id = self._io.read_u4le() + + + def _fetch_instances(self): + pass + + + class PlaylistTreeRow(KaitaiStruct): + """A row that holds a playlist name, ID, indication of whether it + is an ordinary playlist or a folder of other playlists, a link + to its parent folder, and its sort order. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.PlaylistTreeRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.parent_id = self._io.read_u4le() + self._unnamed1 = self._io.read_bytes(4) + self.sort_order = self._io.read_u4le() + self.id = self._io.read_u4le() + self.raw_is_folder = self._io.read_u4le() + self.name = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + + + def _fetch_instances(self): + pass + self.name._fetch_instances() + + @property + def is_folder(self): + if hasattr(self, '_m_is_folder'): + return self._m_is_folder + + self._m_is_folder = self.raw_is_folder != 0 + return getattr(self, '_m_is_folder', None) + + + class RowGroup(KaitaiStruct): + """A group of row indices, which are built backwards from the end + of the page. Holds up to sixteen row offsets, along with a bit + mask that indicates whether each row is actually present in the + table. + """ + def __init__(self, group_index, _io, _parent=None, _root=None): + super(RekordboxPdb.RowGroup, self).__init__(_io) + self._parent = _parent + self._root = _root + self.group_index = group_index + self._read() + + def _read(self): + pass + + + def _fetch_instances(self): + pass + _ = self.row_present_flags + if hasattr(self, '_m_row_present_flags'): + pass + + _ = self.rows + if hasattr(self, '_m_rows'): + pass + for i in range(len(self._m_rows)): + pass + self._m_rows[i]._fetch_instances() + + + _ = self.transaction_row_flags + if hasattr(self, '_m_transaction_row_flags'): + pass + + + @property + def base(self): + """The starting point of this group of row indices. + """ + if hasattr(self, '_m_base'): + return self._m_base + + self._m_base = self._root.len_page - self.group_index * 36 + return getattr(self, '_m_base', None) + + @property + def row_present_flags(self): + """Each bit specifies whether a particular row is present. The + low order bit corresponds to the first row in this index, + whose offset immediately precedes these flag bits. The + second bit corresponds to the row whose offset precedes + that, and so on. + """ + if hasattr(self, '_m_row_present_flags'): + return self._m_row_present_flags + + _pos = self._io.pos() + self._io.seek(self.base - 4) + self._m_row_present_flags = self._io.read_u2le() + self._io.seek(_pos) + return getattr(self, '_m_row_present_flags', None) + + @property + def rows(self): + """The row offsets in this group. + """ + if hasattr(self, '_m_rows'): + return self._m_rows + + self._m_rows = [] + for i in range(16): + self._m_rows.append(RekordboxPdb.RowRef(i, self._io, self, self._root)) + + return getattr(self, '_m_rows', None) + + @property + def transaction_row_flags(self): + """Each bit specifies whether a particular row was touched by + the last transaction on this row group, using the same + layout as row_present_flags. + """ + if hasattr(self, '_m_transaction_row_flags'): + return self._m_transaction_row_flags + + _pos = self._io.pos() + self._io.seek(self.base) + self._m_transaction_row_flags = self._io.read_u2le() + self._io.seek(_pos) + return getattr(self, '_m_transaction_row_flags', None) + + + class RowRef(KaitaiStruct): + """An offset which points to a row in the table, whose actual + presence is controlled by one of the bits in + `row_present_flags`. This instance allows the row itself to be + lazily loaded, unless it is not present, in which case there is + no content to be loaded. + """ + def __init__(self, row_index, _io, _parent=None, _root=None): + super(RekordboxPdb.RowRef, self).__init__(_io) + self._parent = _parent + self._root = _root + self.row_index = row_index + self._read() + + def _read(self): + pass + + + def _fetch_instances(self): + pass + _ = self.body + if hasattr(self, '_m_body'): + pass + _on = self._parent._parent.type + if _on == RekordboxPdb.PageType.albums: + pass + self._m_body._fetch_instances() + elif _on == RekordboxPdb.PageType.artists: + pass + self._m_body._fetch_instances() + elif _on == RekordboxPdb.PageType.artwork: + pass + self._m_body._fetch_instances() + elif _on == RekordboxPdb.PageType.colors: + pass + self._m_body._fetch_instances() + elif _on == RekordboxPdb.PageType.genres: + pass + self._m_body._fetch_instances() + elif _on == RekordboxPdb.PageType.history_entries: + pass + self._m_body._fetch_instances() + elif _on == RekordboxPdb.PageType.history_playlists: + pass + self._m_body._fetch_instances() + elif _on == RekordboxPdb.PageType.keys: + pass + self._m_body._fetch_instances() + elif _on == RekordboxPdb.PageType.labels: + pass + self._m_body._fetch_instances() + elif _on == RekordboxPdb.PageType.playlist_entries: + pass + self._m_body._fetch_instances() + elif _on == RekordboxPdb.PageType.playlist_tree: + pass + self._m_body._fetch_instances() + elif _on == RekordboxPdb.PageType.tracks: + pass + self._m_body._fetch_instances() + + _ = self.body_ext + if hasattr(self, '_m_body_ext'): + pass + _on = self._parent._parent.type_ext + if _on == RekordboxPdb.PageTypeExt.tag_tracks: + pass + self._m_body_ext._fetch_instances() + elif _on == RekordboxPdb.PageTypeExt.tags: + pass + self._m_body_ext._fetch_instances() + + _ = self.ofs_row + if hasattr(self, '_m_ofs_row'): + pass + + + @property + def body(self): + """The actual content of the row, as long as it is present. + """ + if hasattr(self, '_m_body'): + return self._m_body + + if ((self.present) and ((not (self._root.is_ext)))) : + pass + _pos = self._io.pos() + self._io.seek(self.row_base) + _on = self._parent._parent.type + if _on == RekordboxPdb.PageType.albums: + pass + self._m_body = RekordboxPdb.AlbumRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageType.artists: + pass + self._m_body = RekordboxPdb.ArtistRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageType.artwork: + pass + self._m_body = RekordboxPdb.ArtworkRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageType.colors: + pass + self._m_body = RekordboxPdb.ColorRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageType.genres: + pass + self._m_body = RekordboxPdb.GenreRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageType.history_entries: + pass + self._m_body = RekordboxPdb.HistoryEntryRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageType.history_playlists: + pass + self._m_body = RekordboxPdb.HistoryPlaylistRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageType.keys: + pass + self._m_body = RekordboxPdb.KeyRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageType.labels: + pass + self._m_body = RekordboxPdb.LabelRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageType.playlist_entries: + pass + self._m_body = RekordboxPdb.PlaylistEntryRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageType.playlist_tree: + pass + self._m_body = RekordboxPdb.PlaylistTreeRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageType.tracks: + pass + self._m_body = RekordboxPdb.TrackRow(self._io, self, self._root) + self._io.seek(_pos) + + return getattr(self, '_m_body', None) + + @property + def body_ext(self): + """The actual content of the row in an exportExt.pdb file, as long as it is present. + """ + if hasattr(self, '_m_body_ext'): + return self._m_body_ext + + if ((self.present) and (self._root.is_ext)) : + pass + _pos = self._io.pos() + self._io.seek(self.row_base) + _on = self._parent._parent.type_ext + if _on == RekordboxPdb.PageTypeExt.tag_tracks: + pass + self._m_body_ext = RekordboxPdb.TagTrackRow(self._io, self, self._root) + elif _on == RekordboxPdb.PageTypeExt.tags: + pass + self._m_body_ext = RekordboxPdb.TagRow(self._io, self, self._root) + self._io.seek(_pos) + + return getattr(self, '_m_body_ext', None) + + @property + def ofs_row(self): + """The offset of the start of the row (in bytes past the end of + the page header). + """ + if hasattr(self, '_m_ofs_row'): + return self._m_ofs_row + + _pos = self._io.pos() + self._io.seek(self._parent.base - (6 + 2 * self.row_index)) + self._m_ofs_row = self._io.read_u2le() + self._io.seek(_pos) + return getattr(self, '_m_ofs_row', None) + + @property + def present(self): + """Indicates whether the row index considers this row to be + present in the table. Will be `false` if the row has been + deleted. + """ + if hasattr(self, '_m_present'): + return self._m_present + + self._m_present = (True if self._parent.row_present_flags >> self.row_index & 1 != 0 else False) + return getattr(self, '_m_present', None) + + @property + def row_base(self): + """The location of this row relative to the start of the page. + A variety of pointers (such as all device_sql_string values) + are calculated with respect to this position. + """ + if hasattr(self, '_m_row_base'): + return self._m_row_base + + self._m_row_base = self.ofs_row + self._parent._parent.heap_pos + return getattr(self, '_m_row_base', None) + + + class Table(KaitaiStruct): + """Each table is a linked list of pages containing rows of a single + type. This header describes the nature of the table and links to + its pages by index. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.Table, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + if (not (self._root.is_ext)): + pass + self.type = KaitaiStream.resolve_enum(RekordboxPdb.PageType, self._io.read_u4le()) + + if self._root.is_ext: + pass + self.type_ext = KaitaiStream.resolve_enum(RekordboxPdb.PageTypeExt, self._io.read_u4le()) + + self.empty_candidate = self._io.read_u4le() + self.first_page = RekordboxPdb.PageRef(self._io, self, self._root) + self.last_page = RekordboxPdb.PageRef(self._io, self, self._root) + + + def _fetch_instances(self): + pass + if (not (self._root.is_ext)): + pass + + if self._root.is_ext: + pass + + self.first_page._fetch_instances() + self.last_page._fetch_instances() + + + class TagRow(KaitaiStruct): + """A row that holds a tag name and its ID (found only in exportExt.pdb files). + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.TagRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.subtype = self._io.read_u2le() + self.tag_index = self._io.read_u2le() + self._unnamed2 = self._io.read_u8le() + self.category = self._io.read_u4le() + self.category_pos = self._io.read_u4le() + self.id = self._io.read_u4le() + self.raw_is_category = self._io.read_u4le() + self._unnamed7 = self._io.read_u1() + self.ofs_name_near = self._io.read_u1() + self.ofs_unknown_near = self._io.read_u1() + + + def _fetch_instances(self): + pass + _ = self.name + if hasattr(self, '_m_name'): + pass + self._m_name._fetch_instances() + + _ = self.ofs_name_far + if hasattr(self, '_m_ofs_name_far'): + pass + + + @property + def is_category(self): + """Indicates whether this row stores a tag category instead of a tag. + """ + if hasattr(self, '_m_is_category'): + return self._m_is_category + + self._m_is_category = self.raw_is_category != 0 + return getattr(self, '_m_is_category', None) + + @property + def name(self): + """The name of this tag or tag category. + """ + if hasattr(self, '_m_name'): + return self._m_name + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + (self.ofs_name_far if self.subtype & 4 == 4 else self.ofs_name_near)) + self._m_name = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_name', None) + + @property + def ofs_name_far(self): + """For names that might be further than 0xff bytes from the + start of this row, this holds a two-byte offset, and is + signalled by the subtype value. + """ + if hasattr(self, '_m_ofs_name_far'): + return self._m_ofs_name_far + + if self.subtype & 4 == 4: + pass + _pos = self._io.pos() + self._io.seek(self._parent.row_base + 30) + self._m_ofs_name_far = self._io.read_u2le() + self._io.seek(_pos) + + return getattr(self, '_m_ofs_name_far', None) + + + class TagTrackRow(KaitaiStruct): + """A row that associates a track and a tag (found only in exportExt.pdb files). + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.TagTrackRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self._unnamed0 = self._io.read_u4le() + self.track_id = self._io.read_u4le() + self.tag_id = self._io.read_u4le() + self._unnamed3 = self._io.read_u4le() + + + def _fetch_instances(self): + pass + + + class TrackRow(KaitaiStruct): + """A row that describes a track that can be played, with many + details about the music, and links to other tables like artists, + albums, keys, etc. + """ + def __init__(self, _io, _parent=None, _root=None): + super(RekordboxPdb.TrackRow, self).__init__(_io) + self._parent = _parent + self._root = _root + self._read() + + def _read(self): + self.subtype = self._io.read_u2le() + self.index_shift = self._io.read_u2le() + self.bitmask = self._io.read_u4le() + self.sample_rate = self._io.read_u4le() + self.composer_id = self._io.read_u4le() + self.file_size = self._io.read_u4le() + self._unnamed6 = self._io.read_u4le() + self._unnamed7 = self._io.read_u2le() + self._unnamed8 = self._io.read_u2le() + self.artwork_id = self._io.read_u4le() + self.key_id = self._io.read_u4le() + self.original_artist_id = self._io.read_u4le() + self.label_id = self._io.read_u4le() + self.remixer_id = self._io.read_u4le() + self.bitrate = self._io.read_u4le() + self.track_number = self._io.read_u4le() + self.tempo = self._io.read_u4le() + self.genre_id = self._io.read_u4le() + self.album_id = self._io.read_u4le() + self.artist_id = self._io.read_u4le() + self.id = self._io.read_u4le() + self.disc_number = self._io.read_u2le() + self.play_count = self._io.read_u2le() + self.year = self._io.read_u2le() + self.sample_depth = self._io.read_u2le() + self.duration = self._io.read_u2le() + self._unnamed26 = self._io.read_u2le() + self.color_id = self._io.read_u1() + self.rating = self._io.read_u1() + self._unnamed29 = self._io.read_u2le() + self._unnamed30 = self._io.read_u2le() + self.ofs_strings = [] + for i in range(21): + self.ofs_strings.append(self._io.read_u2le()) + + + + def _fetch_instances(self): + pass + for i in range(len(self.ofs_strings)): + pass + + _ = self.analyze_date + if hasattr(self, '_m_analyze_date'): + pass + self._m_analyze_date._fetch_instances() + + _ = self.analyze_path + if hasattr(self, '_m_analyze_path'): + pass + self._m_analyze_path._fetch_instances() + + _ = self.autoload_hot_cues + if hasattr(self, '_m_autoload_hot_cues'): + pass + self._m_autoload_hot_cues._fetch_instances() + + _ = self.comment + if hasattr(self, '_m_comment'): + pass + self._m_comment._fetch_instances() + + _ = self.date_added + if hasattr(self, '_m_date_added'): + pass + self._m_date_added._fetch_instances() + + _ = self.file_path + if hasattr(self, '_m_file_path'): + pass + self._m_file_path._fetch_instances() + + _ = self.filename + if hasattr(self, '_m_filename'): + pass + self._m_filename._fetch_instances() + + _ = self.isrc + if hasattr(self, '_m_isrc'): + pass + self._m_isrc._fetch_instances() + + _ = self.kuvo_public + if hasattr(self, '_m_kuvo_public'): + pass + self._m_kuvo_public._fetch_instances() + + _ = self.message + if hasattr(self, '_m_message'): + pass + self._m_message._fetch_instances() + + _ = self.mix_name + if hasattr(self, '_m_mix_name'): + pass + self._m_mix_name._fetch_instances() + + _ = self.release_date + if hasattr(self, '_m_release_date'): + pass + self._m_release_date._fetch_instances() + + _ = self.texter + if hasattr(self, '_m_texter'): + pass + self._m_texter._fetch_instances() + + _ = self.title + if hasattr(self, '_m_title'): + pass + self._m_title._fetch_instances() + + _ = self.unknown_string_2 + if hasattr(self, '_m_unknown_string_2'): + pass + self._m_unknown_string_2._fetch_instances() + + _ = self.unknown_string_3 + if hasattr(self, '_m_unknown_string_3'): + pass + self._m_unknown_string_3._fetch_instances() + + _ = self.unknown_string_4 + if hasattr(self, '_m_unknown_string_4'): + pass + self._m_unknown_string_4._fetch_instances() + + _ = self.unknown_string_5 + if hasattr(self, '_m_unknown_string_5'): + pass + self._m_unknown_string_5._fetch_instances() + + _ = self.unknown_string_6 + if hasattr(self, '_m_unknown_string_6'): + pass + self._m_unknown_string_6._fetch_instances() + + _ = self.unknown_string_7 + if hasattr(self, '_m_unknown_string_7'): + pass + self._m_unknown_string_7._fetch_instances() + + _ = self.unknown_string_8 + if hasattr(self, '_m_unknown_string_8'): + pass + self._m_unknown_string_8._fetch_instances() + + + @property + def analyze_date(self): + """A string containing the date this track was analyzed by rekordbox. + """ + if hasattr(self, '_m_analyze_date'): + return self._m_analyze_date + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[15]) + self._m_analyze_date = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_analyze_date', None) + + @property + def analyze_path(self): + """The file path of the track analysis, which allows rapid + seeking to particular times in variable bit-rate files, + jumping to particular beats, visual waveform previews, and + stores cue points and loops. + """ + if hasattr(self, '_m_analyze_path'): + return self._m_analyze_path + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[14]) + self._m_analyze_path = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_analyze_path', None) + + @property + def autoload_hot_cues(self): + """A string whose value is always either empty or "ON", and + which apparently for some insane reason is used, rather than + a single bit somewhere, to control whether hot-cues are + auto-loaded for the track. + """ + if hasattr(self, '_m_autoload_hot_cues'): + return self._m_autoload_hot_cues + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[7]) + self._m_autoload_hot_cues = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_autoload_hot_cues', None) + + @property + def comment(self): + """The comment assigned to the track by the DJ, if any. + """ + if hasattr(self, '_m_comment'): + return self._m_comment + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[16]) + self._m_comment = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_comment', None) + + @property + def date_added(self): + """A string containing the date this track was added to the collection. + """ + if hasattr(self, '_m_date_added'): + return self._m_date_added + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[10]) + self._m_date_added = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_date_added', None) + + @property + def file_path(self): + """The file path of the track audio file. + """ + if hasattr(self, '_m_file_path'): + return self._m_file_path + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[20]) + self._m_file_path = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_file_path', None) + + @property + def filename(self): + """The file name of the track audio file. + """ + if hasattr(self, '_m_filename'): + return self._m_filename + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[19]) + self._m_filename = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_filename', None) + + @property + def isrc(self): + """International Standard Recording Code of track + when known (in mangled format). + """ + if hasattr(self, '_m_isrc'): + return self._m_isrc + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[0]) + self._m_isrc = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_isrc', None) + + @property + def kuvo_public(self): + """A string whose value is always either empty or "ON", and + which apparently for some insane reason is used, rather than + a single bit somewhere, to control whether the track + information is visible on Kuvo. + """ + if hasattr(self, '_m_kuvo_public'): + return self._m_kuvo_public + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[6]) + self._m_kuvo_public = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_kuvo_public', None) + + @property + def message(self): + """A string of unknown purpose, which @flesniak named. + """ + if hasattr(self, '_m_message'): + return self._m_message + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[5]) + self._m_message = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_message', None) + + @property + def mix_name(self): + """A string naming the remix of the track, if known. + """ + if hasattr(self, '_m_mix_name'): + return self._m_mix_name + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[12]) + self._m_mix_name = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_mix_name', None) + + @property + def release_date(self): + """A string containing the date this track was released, if known. + """ + if hasattr(self, '_m_release_date'): + return self._m_release_date + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[11]) + self._m_release_date = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_release_date', None) + + @property + def texter(self): + """A string of unknown purpose, which @flesniak named. + """ + if hasattr(self, '_m_texter'): + return self._m_texter + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[1]) + self._m_texter = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_texter', None) + + @property + def title(self): + """The title of the track. + """ + if hasattr(self, '_m_title'): + return self._m_title + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[17]) + self._m_title = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_title', None) + + @property + def unknown_string_2(self): + """A string of unknown purpose; @flesniak said "thought + track number -> wrong!" + """ + if hasattr(self, '_m_unknown_string_2'): + return self._m_unknown_string_2 + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[2]) + self._m_unknown_string_2 = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_unknown_string_2', None) + + @property + def unknown_string_3(self): + """A string of unknown purpose; @flesniak said "strange + strings, often zero length, sometimes low binary values + 0x01/0x02 as content" + """ + if hasattr(self, '_m_unknown_string_3'): + return self._m_unknown_string_3 + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[3]) + self._m_unknown_string_3 = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_unknown_string_3', None) + + @property + def unknown_string_4(self): + """A string of unknown purpose; @flesniak said "strange + strings, often zero length, sometimes low binary values + 0x01/0x02 as content" + """ + if hasattr(self, '_m_unknown_string_4'): + return self._m_unknown_string_4 + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[4]) + self._m_unknown_string_4 = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_unknown_string_4', None) + + @property + def unknown_string_5(self): + """A string of unknown purpose. + """ + if hasattr(self, '_m_unknown_string_5'): + return self._m_unknown_string_5 + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[8]) + self._m_unknown_string_5 = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_unknown_string_5', None) + + @property + def unknown_string_6(self): + """A string of unknown purpose, usually empty. + """ + if hasattr(self, '_m_unknown_string_6'): + return self._m_unknown_string_6 + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[9]) + self._m_unknown_string_6 = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_unknown_string_6', None) + + @property + def unknown_string_7(self): + """A string of unknown purpose, usually empty. + """ + if hasattr(self, '_m_unknown_string_7'): + return self._m_unknown_string_7 + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[13]) + self._m_unknown_string_7 = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_unknown_string_7', None) + + @property + def unknown_string_8(self): + """A string of unknown purpose, usually empty. + """ + if hasattr(self, '_m_unknown_string_8'): + return self._m_unknown_string_8 + + _pos = self._io.pos() + self._io.seek(self._parent.row_base + self.ofs_strings[18]) + self._m_unknown_string_8 = RekordboxPdb.DeviceSqlString(self._io, self, self._root) + self._io.seek(_pos) + return getattr(self, '_m_unknown_string_8', None) + + + diff --git a/maple-py/src/maple/artwork.py b/maple-py/src/maple/artwork.py new file mode 100644 index 0000000..f1de195 --- /dev/null +++ b/maple-py/src/maple/artwork.py @@ -0,0 +1,67 @@ +"""Extract embedded album art and write Pioneer-format JPEGs. + +Pioneer's CDJ/XDJ players expect two square JPEGs per artwork: + /PIONEER/Artwork/{bucket:05d}/a{id}.jpg -> 80x80 + /PIONEER/Artwork/{bucket:05d}/a{id}_m.jpg -> 240x240 +where bucket = id // 20 + 1. The pdb's artwork_row.path points at the +small (80x80) variant; the player auto-resolves the `_m` companion. +""" +import hashlib +import io +from pathlib import Path + +from mutagen.id3 import ID3, ID3NoHeaderError +from PIL import Image + +SMALL = 80 +LARGE = 240 + + +def _read_apic(mp3: Path) -> bytes | None: + try: + tags = ID3(mp3) + except ID3NoHeaderError: + return None + frames = tags.getall("APIC") + if not frames: + return None + return frames[0].data + + +def _bucket(artwork_id: int) -> str: + return f"{artwork_id // 20 + 1:05d}" + + +def _resize_jpeg(data: bytes, size: int) -> bytes: + img = Image.open(io.BytesIO(data)) + if img.mode != "RGB": + img = img.convert("RGB") + img = img.resize((size, size), Image.LANCZOS) + out = io.BytesIO() + img.save(out, format="JPEG", quality=85) + return out.getvalue() + + +def extract_artwork(mp3: Path, usb_root: Path, next_id: int, cache: dict[str, int]) -> tuple[int, str | None]: + """Return (artwork_id, file_path_for_pdb). artwork_id is 0 if no art.""" + data = _read_apic(mp3) + if data is None: + return 0, None + sha = hashlib.sha256(data).hexdigest() + cached = cache.get(sha) + if cached is not None: + return cached, None + + aid = next_id + bucket = _bucket(aid) + rel_small = f"/PIONEER/Artwork/{bucket}/a{aid}.jpg" + rel_large = f"/PIONEER/Artwork/{bucket}/a{aid}_m.jpg" + abs_small = usb_root / rel_small.lstrip("/") + abs_large = usb_root / rel_large.lstrip("/") + abs_small.parent.mkdir(parents=True, exist_ok=True) + if not abs_small.exists(): + abs_small.write_bytes(_resize_jpeg(data, SMALL)) + if not abs_large.exists(): + abs_large.write_bytes(_resize_jpeg(data, LARGE)) + cache[sha] = aid + return aid, rel_small diff --git a/maple-py/src/maple/cli.py b/maple-py/src/maple/cli.py index 134171b..f6c5e10 100644 --- a/maple-py/src/maple/cli.py +++ b/maple-py/src/maple/cli.py @@ -1,12 +1,16 @@ """maple CLI: scan a directory of audio + m3u8 playlists and build a rekordbox USB.""" import argparse +import os import sys +from concurrent.futures import ThreadPoolExecutor from datetime import date from pathlib import Path -from maple.copy import stage_audio +from maple.artwork import extract_artwork +from maple.convert import convert_flac_to_mp3, verify_duration +from maple.copy import stage_audio, staged_path from maple.meta import read_tags -from maple.model import Album, Artist, Database, PlaylistEntry, PlaylistNode, Track +from maple.model import Album, Artist, Artwork, Database, PlaylistEntry, PlaylistNode, Track from maple.pdb.writer import write_pdb from maple.scan import scan_audio, scan_playlists @@ -18,10 +22,30 @@ def build(input_dir: Path, usb_root: Path) -> Database: artist_index: dict[str, int] = {} album_index: dict[tuple[int, str], int] = {} track_by_src: dict[Path, int] = {} + artwork_cache: dict[str, int] = {} audio_files = scan_audio(input_dir) - for src in audio_files: - meta = read_tags(src) + + initial_meta = {p: read_tags(p) for p in audio_files} + flac_jobs: list[tuple[Path, Path]] = [] + for original in audio_files: + if original.suffix.lower() == ".flac": + m = initial_meta[original] + dest_abs, _ = staged_path(usb_root, m.artist, m.album, original.with_suffix(".mp3").name) + flac_jobs.append((original, dest_abs)) + + def _job(pair: tuple[Path, Path]) -> tuple[Path, Path, bool]: + src, dst = pair + return src, dst, convert_flac_to_mp3(src, dst) + + workers = max(1, os.cpu_count() or 1) + with ThreadPoolExecutor(max_workers=workers) as ex: + for src, dst, converted in ex.map(_job, flac_jobs): + print(f"{'convert' if converted else 'cached '} {src.relative_to(input_dir)}") + verify_duration(src, dst) + + for original in audio_files: + meta = initial_meta[original] artist_id = artist_index.get(meta.artist) if artist_id is None: artist_id = len(db.artists) + 1 @@ -35,7 +59,17 @@ def build(input_dir: Path, usb_root: Path) -> Database: album_index[album_key] = album_id db.albums.append(Album(id=album_id, name=meta.album, artist_id=artist_id)) - _, file_path = stage_audio(src, usb_root, meta.artist, meta.album) + if original.suffix.lower() == ".flac": + dest_abs, file_path = staged_path(usb_root, meta.artist, meta.album, original.with_suffix(".mp3").name) + meta = read_tags(dest_abs) + else: + dest_abs, file_path = stage_audio(original, usb_root, meta.artist, meta.album) + + next_artwork_id = len(db.artwork) + 1 + artwork_id, art_file_path = extract_artwork(dest_abs, usb_root, next_artwork_id, artwork_cache) + if art_file_path is not None: + db.artwork.append(Artwork(id=artwork_id, path=art_file_path)) + track_id = len(db.tracks) + 1 db.tracks.append(Track( id=track_id, @@ -52,9 +86,10 @@ def build(input_dir: Path, usb_root: Path) -> Database: track_number=meta.track_number, year=meta.year, date_added=today, - source_path=src, + artwork_id=artwork_id, + source_path=original, )) - track_by_src[src.resolve()] = track_id + track_by_src[original.resolve()] = track_id playlists = scan_playlists(input_dir) for sort_order, (name, paths) in enumerate(playlists.items(), start=1): @@ -82,7 +117,7 @@ def build(input_dir: Path, usb_root: Path) -> Database: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="maple", description="Build a rekordbox USB from a folder of audio + m3u8 playlists.") - parser.add_argument("input_dir", type=Path) + parser.add_argument("input_dir", nargs="?", type=Path, default=Path("./data/input")) parser.add_argument("output_root", nargs="?", type=Path, default=Path("./data/usb")) args = parser.parse_args(argv) diff --git a/maple-py/src/maple/convert.py b/maple-py/src/maple/convert.py new file mode 100644 index 0000000..8177d67 --- /dev/null +++ b/maple-py/src/maple/convert.py @@ -0,0 +1,82 @@ +"""FLAC -> MP3 V0 joint-stereo conversion via ffmpeg, with source-hash dedup. + +The SHA-256 of the source FLAC is stamped into a TXXX:MAPLE_SRC_SHA256 ID3 +frame on the produced MP3. Reruns skip conversion when the existing MP3 +carries the matching hash, so the cache is content-addressed against inputs. +""" +import hashlib +import shutil +import subprocess +from pathlib import Path + +from mutagen import File as MutagenFile +from mutagen.id3 import ID3, ID3NoHeaderError, TXXX + +HASH_FRAME_DESC = "MAPLE_SRC_SHA256" + + +def file_sha256(path: Path) -> str: + with path.open("rb") as f: + return hashlib.file_digest(f, "sha256").hexdigest() + + +def _read_source_hash(mp3: Path) -> str | None: + try: + tags = ID3(mp3) + except ID3NoHeaderError: + return None + for frame in tags.getall("TXXX"): + if frame.desc == HASH_FRAME_DESC and frame.text: + return str(frame.text[0]) + return None + + +def _write_source_hash(mp3: Path, sha: str) -> None: + try: + tags = ID3(mp3) + except ID3NoHeaderError: + tags = ID3() + tags.delall(f"TXXX:{HASH_FRAME_DESC}") + tags.add(TXXX(encoding=3, desc=HASH_FRAME_DESC, text=sha)) + tags.save(mp3) + + +def convert_flac_to_mp3(flac: Path, mp3: Path) -> bool: + """Convert flac to mp3 (V0, joint stereo). Returns True if conversion ran.""" + src_hash = file_sha256(flac) + if mp3.exists() and _read_source_hash(mp3) == src_hash: + return False + + if not shutil.which("ffmpeg"): + raise RuntimeError("ffmpeg not found on PATH") + + mp3.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + "ffmpeg", "-nostdin", "-loglevel", "error", "-y", + "-i", str(flac), + "-codec:a", "libmp3lame", + "-q:a", "0", + "-joint_stereo", "1", + "-map_metadata", "0", + str(mp3), + ], + check=True, + ) + _write_source_hash(mp3, src_hash) + return True + + +def _duration(path: Path) -> float: + audio = MutagenFile(path) + info = getattr(audio, "info", None) + return float(getattr(info, "length", 0.0) or 0.0) + + +def verify_duration(src: Path, dst: Path, tolerance: float = 0.1) -> None: + """Raise if dst's audio length differs from src's by more than `tolerance` seconds.""" + a, b = _duration(src), _duration(dst) + if abs(a - b) > tolerance: + raise RuntimeError( + f"duration mismatch: {src.name} = {a:.3f}s vs {dst.name} = {b:.3f}s (Δ {abs(a-b):.3f}s)" + ) diff --git a/maple-py/src/maple/copy.py b/maple-py/src/maple/copy.py index cf825a1..82ab0a2 100644 --- a/maple-py/src/maple/copy.py +++ b/maple-py/src/maple/copy.py @@ -1,24 +1,66 @@ -"""Copy audio files into the USB Contents layout.""" +"""Copy audio files into the USB Contents layout. + +Targets are assumed to be FAT32-formatted USBs, which constrain us to: + - per-component name length <= 255 LFN chars + - total path (from drive root) <= 260 chars + - no '\\ / : * ? " < > |' or control chars in any component +We truncate per-component to a safe 64 chars (rekordbox firmware is +fussier than raw FAT32) and refuse to emit anything that would still +overflow the 260-char path budget. +""" import re import shutil from pathlib import Path _BAD = re.compile(r'[\\/:*?"<>|\x00-\x1f]') +MAX_COMPONENT = 48 +MAX_PATH = 260 +# Pioneer firmware appears to only handle the DeviceSQL short-ASCII form +# for `file_path` in the pdb (length byte < 127). The verified reference +# export uses short-form for every track. Keep the on-pdb path strictly +# inside that budget so every track stays loadable. +MAX_PDB_PATH = 126 -def sanitise(component: str) -> str: + +def sanitise(component: str, max_len: int = MAX_COMPONENT) -> str: s = _BAD.sub("_", component).strip(" .") + if len(s) > max_len: + s = s[:max_len].rstrip(" .") return s or "_" -def stage_audio(src: Path, usb_root: Path, artist: str, album: str) -> tuple[Path, str]: - artist_dir = sanitise(artist) - album_dir = sanitise(album) - filename = sanitise(src.name) - dest_rel = Path("Contents") / artist_dir / album_dir / filename +def _truncate_filename(name: str, max_len: int) -> str: + """Truncate while preserving the extension.""" + cleaned = _BAD.sub("_", name).strip(" .") + if len(cleaned) <= max_len: + return cleaned or "_" + stem, dot, ext = cleaned.rpartition(".") + if dot and len(ext) < max_len - 1: + keep = max_len - len(ext) - 1 + return (stem[:keep].rstrip(" .") + "." + ext) or "_" + return cleaned[:max_len].rstrip(" .") or "_" + + +def staged_path(usb_root: Path, artist: str, album: str, filename: str) -> tuple[Path, str]: + artist_s = sanitise(artist) + album_s = sanitise(album) + prefix = f"/Contents/{artist_s}/{album_s}/" + budget = min(MAX_COMPONENT, MAX_PDB_PATH - len(prefix)) + if budget < 8: + raise ValueError(f"no room for filename within MAX_PDB_PATH={MAX_PDB_PATH} (prefix uses {len(prefix)}): {prefix}") + name_s = _truncate_filename(filename, budget) + dest_rel = Path("Contents") / artist_s / album_s / name_s dest_abs = usb_root / dest_rel + file_path = "/" + dest_rel.as_posix() + if len(file_path) > MAX_PDB_PATH: + raise ValueError(f"pdb path budget exceeded ({len(file_path)} > {MAX_PDB_PATH}): {file_path}") + return dest_abs, file_path + + +def stage_audio(src: Path, usb_root: Path, artist: str, album: str) -> tuple[Path, str]: + dest_abs, file_path = staged_path(usb_root, artist, album, src.name) dest_abs.parent.mkdir(parents=True, exist_ok=True) if not dest_abs.exists(): shutil.copy2(src, dest_abs) - file_path = "/" + dest_rel.as_posix() return dest_abs, file_path diff --git a/maple-py/src/maple/model.py b/maple-py/src/maple/model.py index 4f7591a..76bf551 100644 --- a/maple-py/src/maple/model.py +++ b/maple-py/src/maple/model.py @@ -39,9 +39,57 @@ class Track: mix_name: str = "" analyze_path: str = "" analyze_date: str = "" + artwork_id: int = 0 source_path: Path | None = None +@dataclass +class Artwork: + id: int + path: str + + +@dataclass +class Column: + id: int + code: int + name: str + + +# Reverse-engineered from a verified rekordbox export. Each entry is +# (column_id, internal column code, display name). Code values are +# non-contiguous because rekordbox skips reserved slots. +DEFAULT_COLUMNS: list[Column] = [ + Column(1, 0x80, "GENRE"), + Column(2, 0x81, "ARTIST"), + Column(3, 0x82, "ALBUM"), + Column(4, 0x83, "TRACK"), + Column(5, 0x85, "BPM"), + Column(6, 0x86, "RATING"), + Column(7, 0x87, "YEAR"), + Column(8, 0x88, "REMIXER"), + Column(9, 0x89, "LABEL"), + Column(10, 0x8A, "ORIGINAL ARTIST"), + Column(11, 0x8B, "KEY"), + Column(12, 0x8D, "CUE"), + Column(13, 0x8E, "COLOR"), + Column(14, 0x92, "TIME"), + Column(15, 0x93, "BITRATE"), + Column(16, 0x94, "FILE NAME"), + Column(17, 0x84, "PLAYLIST"), + Column(18, 0x98, "HOT CUE BANK"), + Column(19, 0x95, "HISTORY"), + Column(20, 0x91, "SEARCH"), + Column(21, 0x96, "COMMENTS"), + Column(22, 0x8C, "DATE ADDED"), + Column(23, 0x97, "DJ PLAY COUNT"), + Column(24, 0x90, "FOLDER"), + Column(25, 0xA1, "DEFAULT"), + Column(26, 0xA2, "ALPHABET"), + Column(27, 0xAA, "MATCHING"), +] + + @dataclass class PlaylistNode: id: int @@ -65,3 +113,4 @@ class Database: albums: list[Album] = field(default_factory=list) playlist_nodes: list[PlaylistNode] = field(default_factory=list) playlist_entries: list[PlaylistEntry] = field(default_factory=list) + artwork: list[Artwork] = field(default_factory=list) diff --git a/maple-py/src/maple/pdb/pager.py b/maple-py/src/maple/pdb/pager.py index a98e3ac..a752a2b 100644 --- a/maple-py/src/maple/pdb/pager.py +++ b/maple-py/src/maple/pdb/pager.py @@ -4,13 +4,42 @@ from struct import pack PAGE_SIZE = 4096 HEADER_SIZE = 40 ROW_GROUP_SIZE = 36 -MAX_ROWS_PER_PAGE = 16 # one row group per page, matching rekordcrate +ROWS_PER_GROUP = 16 def _align4(offset: int) -> int: return (offset + 3) & ~3 +def _num_row_groups(n: int) -> int: + return (n + ROWS_PER_GROUP - 1) // ROWS_PER_GROUP + + +def build_strange_page(page_index: int, page_type: int, next_page: int) -> bytes: + """Empty placeholder page that the reference exports use as the first + page of every table chain. Reproduces the observed header flags and + marker bytes; the heap is left zero-filled (rekordbox fills it with a + `f8 ff ff 1f` pattern, but players don't care because num_rows=0). + """ + header = b"" + header += pack(" bool: - if len(self.row_bytes) >= MAX_ROWS_PER_PAGE: - return False aligned = _align4(self.heap_cursor) end = aligned + len(row) - heap_capacity = PAGE_SIZE - HEADER_SIZE - ROW_GROUP_SIZE - if end > heap_capacity: + footer_after = _num_row_groups(len(self.row_bytes) + 1) * ROW_GROUP_SIZE + if end + HEADER_SIZE + footer_after > PAGE_SIZE: return False self.row_offsets.append(aligned) self.row_bytes.append(row) @@ -38,17 +65,17 @@ class PageBuilder: def finalize(self) -> bytes: n = len(self.row_bytes) - if n == 0: - used_size = 0 - heap_padding = PAGE_SIZE - HEADER_SIZE - else: - used_size = self.heap_cursor - heap_padding = PAGE_SIZE - HEADER_SIZE - ROW_GROUP_SIZE + num_rgs = _num_row_groups(n) + footer_size = num_rgs * ROW_GROUP_SIZE + heap_padding = PAGE_SIZE - HEADER_SIZE - footer_size + used_size = self.heap_cursor free_size = heap_padding - used_size if free_size < 0: raise ValueError("page overflow") - page_flags = 0x00 if n > 0 else 0x44 + # Reference exports use 0x24 for data pages with rows. Empty data + # pages (post-strange) keep 0x44 to mark them as having no rows. + page_flags = 0x24 if n > 0 else 0x44 header = b"" header += pack("> 8) & 0xFF, (packed >> 16) & 0xFF, page_flags]) header += pack(" 0: - for ofs, row in zip(self.row_offsets, self.row_bytes): - heap[ofs:ofs + len(row)] = row + for ofs, row in zip(self.row_offsets, self.row_bytes): + heap[ofs:ofs + len(row)] = row - if n == 0: - return header + bytes(heap) + # Footers go after the heap in the file. Group 0 (rows 0-15) sits closest + # to the end of the page; group num_rgs-1 sits closest to the heap. So + # emit footers in reverse group order. + footer = b"" + for g in range(num_rgs - 1, -1, -1): + start = g * ROWS_PER_GROUP + grp_offsets = self.row_offsets[start:start + ROWS_PER_GROUP] + grp_count = len(grp_offsets) + padded = grp_offsets + [0] * (ROWS_PER_GROUP - grp_count) + presence = (1 << grp_count) - 1 + rg = b"" + for ofs in reversed(padded): + rg += pack(" bytes: prefix += pack(" bytes: def serialize_playlist_entry(e: PlaylistEntry) -> bytes: return pack(" bytes: + row = pack(" bytes: + """A column row is: u2 id, u2 code, then a 'long' devicesql string whose + UTF-16 body is wrapped by the sentinel codepoints U+FFFA / U+FFFB. + """ + body = b"\xfa\xff" + c.name.encode("utf-16-le") + b"\xfb\xff" + total_length = len(body) + 4 # 4 = 0x90 marker + u2 length + u1 pad + name_str = b"\x90" + pack(" short ASCII (< 127 chars) + - 0x40 -> long ASCII (any length, ASCII bytes) + - 0x90 -> long UTF-16LE (for non-ASCII content) + +We pick the cheapest form that fits the content. Long-but-ASCII paths +must use 0x40, NOT 0x90 -- some player firmwares decode 0x90 strings +as UTF-16 and then fail the case-sensitive filesystem lookup, so any +track whose `file_path` lands in the wrong form silently goes missing. +""" EMPTY = b"\x03" @@ -10,10 +21,15 @@ def encode_devicesql(s: str) -> bytes: ascii_bytes = s.encode("ascii") except UnicodeEncodeError: ascii_bytes = None - if ascii_bytes is not None and len(ascii_bytes) < 127: - length = len(ascii_bytes) + 1 - length_and_kind = ((length << 1) | 1) & 0xFF - return bytes([length_and_kind]) + ascii_bytes + + if ascii_bytes is not None: + if len(ascii_bytes) < 127: + length = len(ascii_bytes) + 1 + length_and_kind = ((length << 1) | 1) & 0xFF + return bytes([length_and_kind]) + ascii_bytes + total_length = len(ascii_bytes) + 4 # marker + u2 length + u1 pad + return bytes([0x40]) + total_length.to_bytes(2, "little") + b"\x00" + ascii_bytes + payload = s.encode("utf-16-le") total_length = len(payload) + 4 return bytes([0x90]) + total_length.to_bytes(2, "little") + b"\x00" + payload diff --git a/maple-py/src/maple/pdb/writer.py b/maple-py/src/maple/pdb/writer.py index 801fed5..6bfb9f5 100644 --- a/maple-py/src/maple/pdb/writer.py +++ b/maple-py/src/maple/pdb/writer.py @@ -2,56 +2,86 @@ from pathlib import Path from struct import pack -from maple.model import Database -from maple.pdb.pager import PAGE_SIZE, PageBuilder +from maple.model import DEFAULT_COLUMNS, Database +from maple.pdb.pager import PAGE_SIZE, PageBuilder, build_strange_page from maple.pdb.rows import ( serialize_album, serialize_artist, + serialize_artwork, + serialize_column, serialize_playlist_entry, serialize_playlist_tree, serialize_track, ) PAGE_TYPE_TRACKS = 0 +PAGE_TYPE_GENRES = 1 PAGE_TYPE_ARTISTS = 2 PAGE_TYPE_ALBUMS = 3 +PAGE_TYPE_LABELS = 4 +PAGE_TYPE_KEYS = 5 +PAGE_TYPE_COLORS = 6 PAGE_TYPE_PLAYLIST_TREE = 7 PAGE_TYPE_PLAYLIST_ENTRIES = 8 +PAGE_TYPE_UNKNOWN_9 = 9 +PAGE_TYPE_UNKNOWN_10 = 10 +PAGE_TYPE_HISTORY_PLAYLISTS = 11 +PAGE_TYPE_HISTORY_ENTRIES = 12 +PAGE_TYPE_ARTWORK = 13 +PAGE_TYPE_UNKNOWN_14 = 14 +PAGE_TYPE_UNKNOWN_15 = 15 +PAGE_TYPE_COLUMNS = 16 +PAGE_TYPE_UNKNOWN_17 = 17 +PAGE_TYPE_UNKNOWN_18 = 18 +PAGE_TYPE_HISTORY = 19 + +ALL_PAGE_TYPES = list(range(20)) FILE_HEADER_SIZE = 28 # 0,page_size,num_tables,next_unused,unknown,sequence,gap = 7*u4 TABLE_ENTRY_SIZE = 16 # page_type,empty_candidate,first_page,last_page = 4*u4 def _build_chain(page_type: int, row_iter, start_index: int) -> tuple[list[bytes], int, int]: - """Return (page_bytes_list, first_page, last_page).""" - pages: list[PageBuilder] = [] - page_idx = start_index - current = PageBuilder(page_idx, page_type) + """Return (page_bytes_list, first_page, last_page). - for row in row_iter: + Every table chain begins with a "strange" placeholder page (flags=0x64) + mirroring rekordbox's own exports. Data pages with rows follow. + Tables with no rows emit only the strange page. + """ + rows = list(row_iter) + out: list[bytes] = [] + + # Reserve start_index for the strange page; data pages follow. + if not rows: + # Strange page only: it points to itself (end of chain). + out.append(build_strange_page(start_index, page_type, next_page=start_index)) + return out, start_index, start_index + + # Build data pages starting at start_index + 1. + data_pages: list[PageBuilder] = [] + page_idx = start_index + 1 + current = PageBuilder(page_idx, page_type) + for row in rows: if not current.try_add(row): - pages.append(current) + data_pages.append(current) page_idx += 1 current = PageBuilder(page_idx, page_type) if not current.try_add(row): raise ValueError(f"row of size {len(row)} too large for an empty page") + data_pages.append(current) - pages.append(current) + # Strange page links to the first data page. + out.append(build_strange_page(start_index, page_type, next_page=start_index + 1)) - # If the chain has no rows at all, still emit one empty page - first_page = start_index - last_page = page_idx - - # Patch next_page pointers: each non-last page points to the next; last points to itself - out: list[bytes] = [] - for i, p in enumerate(pages): - if i == len(pages) - 1: + # Patch each data page's next_page; last points to itself. + for i, p in enumerate(data_pages): + if i == len(data_pages) - 1: p.next_page = p.page_index else: p.next_page = p.page_index + 1 out.append(p.finalize()) - return out, first_page, last_page + return out, start_index, page_idx def write_pdb(db: Database, path: str | Path) -> None: @@ -64,14 +94,24 @@ def write_pdb(db: Database, path: str | Path) -> None: serialized_albums = [serialize_album(a) for a in db.albums] serialized_pl_tree = [serialize_playlist_tree(n) for n in db.playlist_nodes] serialized_pl_entries = [serialize_playlist_entry(e) for e in db.playlist_entries] + serialized_artwork = [serialize_artwork(a) for a in db.artwork] + serialized_columns = [serialize_column(c) for c in DEFAULT_COLUMNS] - for ptype, rows in [ - (PAGE_TYPE_TRACKS, serialized_tracks), - (PAGE_TYPE_ARTISTS, serialized_artists), - (PAGE_TYPE_ALBUMS, serialized_albums), - (PAGE_TYPE_PLAYLIST_TREE, serialized_pl_tree), - (PAGE_TYPE_PLAYLIST_ENTRIES, serialized_pl_entries), - ]: + rows_by_type: dict[int, list[bytes]] = { + PAGE_TYPE_TRACKS: serialized_tracks, + PAGE_TYPE_ARTISTS: serialized_artists, + PAGE_TYPE_ALBUMS: serialized_albums, + PAGE_TYPE_PLAYLIST_TREE: serialized_pl_tree, + PAGE_TYPE_PLAYLIST_ENTRIES: serialized_pl_entries, + PAGE_TYPE_ARTWORK: serialized_artwork, + PAGE_TYPE_COLUMNS: serialized_columns, + } + # Emit all 20 page-type chains; tables we don't populate get a single empty page. + # CDJ/XDJ firmware looks up tables by index, so emitting the full set keeps + # features that probe optional tables (artwork, keys, colors, history) from + # failing the lookup. + for ptype in ALL_PAGE_TYPES: + rows = rows_by_type.get(ptype, []) pages, first, last = _build_chain(ptype, rows, next_page_idx) chains.append((ptype, pages, first, last)) next_page_idx = last + 1 diff --git a/maple-py/uv.lock b/maple-py/uv.lock index 94f3910..4fa3b8b 100644 --- a/maple-py/uv.lock +++ b/maple-py/uv.lock @@ -19,12 +19,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484 }, ] +[[package]] +name = "kaitaistruct" +version = "0.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/b8/ca7319556912f68832daa4b81425314857ec08dfccd8dbc8c0f65c992108/kaitaistruct-0.11.tar.gz", hash = "sha256:053ee764288e78b8e53acf748e9733268acbd579b8d82a427b1805453625d74b", size = 11519 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/4a/cf14bf3b1f5ffb13c69cf5f0ea78031247790558ee88984a8bdd22fae60d/kaitaistruct-0.11-py2.py3-none-any.whl", hash = "sha256:5c6ce79177b4e193a577ecd359e26516d1d6d000a0bffd6e1010f2a46a62a561", size = 11372 }, +] + [[package]] name = "maple" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "kaitaistruct" }, { name = "mutagen" }, + { name = "pillow" }, ] [package.dev-dependencies] @@ -33,7 +44,11 @@ dev = [ ] [package.metadata] -requires-dist = [{ name = "mutagen", specifier = ">=1.47.0" }] +requires-dist = [ + { name = "kaitaistruct", specifier = ">=0.11" }, + { name = "mutagen", specifier = ">=1.47.0" }, + { name = "pillow", specifier = ">=12.2.0" }, +] [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=9.0.3" }] @@ -56,6 +71,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195 }, ] +[[package]] +name = "pillow" +version = "12.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/98/4595daa2365416a86cb0d495248a393dfc84e96d62ad080c8546256cb9c0/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8", size = 4100848 }, + { url = "https://files.pythonhosted.org/packages/0b/79/40184d464cf89f6663e18dfcf7ca21aae2491fff1a16127681bf1fa9b8cf/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b", size = 4176515 }, + { url = "https://files.pythonhosted.org/packages/b0/63/703f86fd4c422a9cf722833670f4f71418fb116b2853ff7da722ea43f184/pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295", size = 3640159 }, + { url = "https://files.pythonhosted.org/packages/71/e0/fb22f797187d0be2270f83500aab851536101b254bfa1eae10795709d283/pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed", size = 5312185 }, + { url = "https://files.pythonhosted.org/packages/ba/8c/1a9e46228571de18f8e28f16fabdfc20212a5d019f3e3303452b3f0a580d/pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae", size = 4695386 }, + { url = "https://files.pythonhosted.org/packages/70/62/98f6b7f0c88b9addd0e87c217ded307b36be024d4ff8869a812b241d1345/pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601", size = 6280384 }, + { url = "https://files.pythonhosted.org/packages/5e/03/688747d2e91cfbe0e64f316cd2e8005698f76ada3130d0194664174fa5de/pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be", size = 8091599 }, + { url = "https://files.pythonhosted.org/packages/f6/35/577e22b936fcdd66537329b33af0b4ccfefaeabd8aec04b266528cddb33c/pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f", size = 6396021 }, + { url = "https://files.pythonhosted.org/packages/11/8d/d2532ad2a603ca2b93ad9f5135732124e57811d0168155852f37fbce2458/pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286", size = 7083360 }, + { url = "https://files.pythonhosted.org/packages/5e/26/d325f9f56c7e039034897e7380e9cc202b1e368bfd04d4cbe6a441f02885/pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50", size = 6507628 }, + { url = "https://files.pythonhosted.org/packages/5f/f7/769d5632ffb0988f1c5e7660b3e731e30f7f8ec4318e94d0a5d674eb65a4/pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104", size = 7209321 }, + { url = "https://files.pythonhosted.org/packages/6a/7a/c253e3c645cd47f1aceea6a8bacdba9991bf45bb7dfe927f7c893e89c93c/pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7", size = 6479723 }, + { url = "https://files.pythonhosted.org/packages/cd/8b/601e6566b957ca50e28725cb6c355c59c2c8609751efbecd980db44e0349/pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150", size = 7217400 }, + { url = "https://files.pythonhosted.org/packages/d6/94/220e46c73065c3e2951bb91c11a1fb636c8c9ad427ac3ce7d7f3359b9b2f/pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1", size = 2554835 }, + { url = "https://files.pythonhosted.org/packages/b6/ab/1b426a3974cb0e7da5c29ccff4807871d48110933a57207b5a676cccc155/pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463", size = 5314225 }, + { url = "https://files.pythonhosted.org/packages/19/1e/dce46f371be2438eecfee2a1960ee2a243bbe5e961890146d2dee1ff0f12/pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3", size = 4698541 }, + { url = "https://files.pythonhosted.org/packages/55/c3/7fbecf70adb3a0c33b77a300dc52e424dc22ad8cdc06557a2e49523b703d/pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166", size = 6322251 }, + { url = "https://files.pythonhosted.org/packages/1c/3c/7fbc17cfb7e4fe0ef1642e0abc17fc6c94c9f7a16be41498e12e2ba60408/pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe", size = 8127807 }, + { url = "https://files.pythonhosted.org/packages/ff/c3/a8ae14d6defd2e448493ff512fae903b1e9bd40b72efb6ec55ce0048c8ce/pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd", size = 6433935 }, + { url = "https://files.pythonhosted.org/packages/6e/32/2880fb3a074847ac159d8f902cb43278a61e85f681661e7419e6596803ed/pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e", size = 7116720 }, + { url = "https://files.pythonhosted.org/packages/46/87/495cc9c30e0129501643f24d320076f4cc54f718341df18cc70ec94c44e1/pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06", size = 6540498 }, + { url = "https://files.pythonhosted.org/packages/18/53/773f5edca692009d883a72211b60fdaf8871cbef075eaa9d577f0a2f989e/pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43", size = 7239413 }, + { url = "https://files.pythonhosted.org/packages/c9/e4/4b64a97d71b2a83158134abbb2f5bd3f8a2ea691361282f010998f339ec7/pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354", size = 6482084 }, + { url = "https://files.pythonhosted.org/packages/ba/13/306d275efd3a3453f72114b7431c877d10b1154014c1ebbedd067770d629/pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1", size = 7225152 }, + { url = "https://files.pythonhosted.org/packages/ff/6e/cf826fae916b8658848d7b9f38d88da6396895c676e8086fc0988073aaf8/pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb", size = 2556579 }, +] + [[package]] name = "pluggy" version = "1.6.0"