i know kung fu

maple-rs/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2024"
 
 [dependencies]
 # rekordcrate = { git = "ssh://git@github.com/ntwl-xyz/rekordcrate.git", branch = "db" }
-rekordcrate = { path = "../../rekordcrate" }
+rekordcrate = { path = "../../code/rekordcrate" }
 lofty = "0.21"
 walkdir = "2.4"
 binrw = "0.15"

maple-rs/src/db.rs

@@ -1,10 +1,30 @@
+use rekordcrate::pdb::Artwork;
+use rekordcrate::pdb::ArtworkId;
+use rekordcrate::pdb::Page;
+use rekordcrate::pdb::PageType;
+use rekordcrate::pdb::PageIndex;
+use std::io::Write;
+use std::io::{Seek, SeekFrom};
+use binrw::BinWrite;
+use rekordcrate::pdb::LabelId;
+use rekordcrate::pdb::KeyId;
+use rekordcrate::pdb::GenreId;
+use rekordcrate::pdb::ArtistId;
+use rekordcrate::pdb::AlbumId;
+use rekordcrate::pdb::Album;
+use rekordcrate::pdb::Artist;
+use rekordcrate::pdb::Genre;
+use rekordcrate::pdb::Key;
+use rekordcrate::pdb::Label;
+use rekordcrate::pdb::Table;
+use std::fs::File;
 use std::path::Path;
 use binrw::BinRead;
 use std::collections::BTreeMap;
 
 use binrw::Endian;
 use rekordcrate::pdb::{
-    Header, PageType, Row,
+    Header, Row,
     Track, TrackId,
     PlaylistTreeNode, PlaylistTreeNodeId,
     PlaylistEntry,
@@ -21,17 +41,149 @@ pub enum DbError {
 pub struct Database {
     pub header: Header,
     pub tracks: BTreeMap<TrackId, Track>,
+    pub albums: BTreeMap<AlbumId, Album>,
+    pub artists: BTreeMap<ArtistId, Artist>,
+    pub artwork: BTreeMap<ArtworkId, Artwork>,
+    pub genres: BTreeMap<GenreId, Genre>,
+    pub keys: BTreeMap<KeyId, Key>,
+    pub labels: BTreeMap<LabelId, Label>,
     pub nodes: BTreeMap<PlaylistTreeNodeId, PlaylistTreeNode>,
     // Keep playlist entries ordered by entry_index
     pub entries: BTreeMap<PlaylistTreeNodeId, Vec<PlaylistEntry>>,
 }
 
+const PAGE_SIZE: u32 = 4096;
+
 impl Database {
+    pub fn export(self, path: &Path) -> std::io::Result<()> {
+        // 1) Reserve header page
+        let mut f = File::create(path)?;
+        f.write_all(&vec![0u8; PAGE_SIZE as usize])?;
+
+        // 2) Emit tables linearly, collect table metadata
+        let mut next_page_index: u32 = 1;
+
+        // ok now we're getting somewhere
+        // smash rows into row_groups of 16
+        // try to write those into a page until it is full
+        // yield pages until all the rows are in
+        // set the indices
+        // 420 blaze it
+
+        let (tracks_first, tracks_last) =
+            Self::write_table(&mut f, PageType::Tracks, next_page_index, self.tracks.values().cloned().map(Row::Track))?;
+        next_page_index = tracks_last + 1;
+
+        let (tree_first, tree_last) =
+            Self::write_table(&mut f, PageType::PlaylistTree, next_page_index, self.nodes.values().cloned().map(Row::PlaylistTreeNode))?;
+        next_page_index = tree_last + 1;
+
+        let plist_entries = self.entries.iter().flat_map(|(_pid, vec)| {
+            // ensure contiguous entry_index
+            let mut v = vec.clone();
+            v.sort_by_key(|e| e.entry_index);
+            v.into_iter().map(Row::PlaylistEntry)
+        });
+        let (entries_first, entries_last) =
+            Self::write_table(&mut f, PageType::PlaylistEntries, next_page_index, plist_entries)?;
+        next_page_index = entries_last + 1;
+
+        let (art_first, art_last) =
+            Self::write_table(&mut f, PageType::Artwork, next_page_index, self.artwork.values().cloned().map(Row::Artwork))?;
+        next_page_index = art_last + 1;
+
+        // 3) Build header and back-patch at page 0
+        let header = Header {
+            // fill other required fields in your crate
+            sequence: 1, // or previous+1
+            next_unused_page: PageIndex(next_page_index),
+            tables: vec![
+                Table { page_type: PageType::Tracks,          first_page: PageIndex(tracks_first),  last_page: PageIndex(tracks_last),  empty_candidate: 0, /* … */ },
+                Table { page_type: PageType::PlaylistTree,    first_page: PageIndex(tree_first),    last_page: PageIndex(tree_last),    empty_candidate: 0, /* … */ },
+                Table { page_type: PageType::PlaylistEntries, first_page: PageIndex(entries_first), last_page: PageIndex(entries_last), empty_candidate: 0, /* … */ },
+                Table { page_type: PageType::Artwork,         first_page: PageIndex(art_first),     last_page: PageIndex(art_last),     empty_candidate: 0, /* … */ },
+            ],
+            ..Default::default()
+        };
+
+        // Write header at offset 0 and pad to one page
+        f.seek(SeekFrom::Start(0))?;
+        header.write(&mut f);
+        // pad residual of page 0 if BinWrite didn’t fill it:
+        let pos = f.stream_position()?;
+        if pos < PAGE_SIZE as u64 {
+            f.write_all(&vec![0u8; (PAGE_SIZE as u64 - pos) as usize])?;
+        }
+        Ok(())
+    }
+
+    fn write_table<I>(
+        f: &mut File,
+        page_type: PageType,
+        start_index: u32,
+        rows: I,
+    ) -> std::io::Result<(u32, u32)>
+    where
+        I: IntoIterator<Item = Row>,
+    {
+        let mut page_index = start_index;
+        let mut first = 0;
+        let mut last = 0;
+
+        // Pack rows into pages. Use your crate’s Page builder to handle row groups.
+        let mut packer = PagePacker::new(page_type, PAGE_SIZE);
+
+        for row in rows {
+            if !packer.try_push(&row) {
+                // flush full page
+                let mut page = packer.finish(page_index, /*next_page=*/0);
+                // Temporarily set next_page=0; we will fix it after writing the next page.
+                Self::write_page(f, &mut page)?;
+                if first == 0 { first = page_index; }
+                last = page_index;
+                page_index += 1;
+                // start new page with the row that didn’t fit
+                packer = PagePacker::new(page_type, PAGE_SIZE);
+                assert!(packer.try_push(&row));
+            }
+        }
+
+        // flush final page if it has content or if this is an empty table create one empty page
+        if packer.is_empty() && first == 0 {
+            let mut page = Page::empty(page_type, page_index, /*next_page=*/0, PAGE_SIZE);
+            Self::write_page(f, &mut page)?;
+            first = page_index;
+            last = page_index;
+        } else {
+            let mut page = packer.finish(page_index, /*next_page=*/0);
+            Self::write_page(f, &mut page)?;
+            if first == 0 { first = page_index; }
+            last = page_index;
+        }
+
+        // Fix `next_page` pointers by rewriting page headers if your Page writer requires it.
+        // Simpler: during emission, remember the previous page file offset and seek back
+        // to patch its `next_page` to `page_index`.
+
+        Ok((first, last))
+    }
+
+    fn write_page(f: &mut File, page: &mut Page) -> std::io::Result<()> {
+        page.write(f);
+        Ok(())
+    }
+
     pub fn load_from_pdb(path: impl AsRef<Path>) -> rekordcrate::Result<Self> {
         let mut f = std::fs::File::open(path)?;
         let header = Header::read(&mut f)?; // uses the type’s endianness
 
         let mut tracks: BTreeMap<TrackId, Track> = BTreeMap::new();
+        let mut albums: BTreeMap<AlbumId, Album> = BTreeMap::new();
+        let mut artists: BTreeMap<ArtistId, Artist> = BTreeMap::new();
+        let mut genres: BTreeMap<GenreId, Genre> = BTreeMap::new();
+        let mut keys: BTreeMap<KeyId, Key> = BTreeMap::new();
+        let mut labels: BTreeMap<LabelId, Label> = BTreeMap::new();
+
         let mut nodes: BTreeMap<PlaylistTreeNodeId, PlaylistTreeNode> = BTreeMap::new();
         let mut entries: BTreeMap<PlaylistTreeNodeId, Vec<PlaylistEntry>> = BTreeMap::new();
 
@@ -45,6 +197,21 @@ impl Database {
                             Row::Track(t) => {
                                 tracks.insert(t.id, t.clone());
                             }
+                            Row::Artist(a) => {
+                                artists.insert(a.id, a.clone());
+                            }
+                            Row::Album(a) => {
+                                albums.insert(a.id, a.clone());
+                            }
+                            Row::Genre(g) => {
+                                genres.insert(g.id, g.clone());
+                            }
+                            Row::Key(k) => {
+                                keys.insert(k.id, k.clone());
+                            }
+                            Row::Label(l) => {
+                                labels.insert(l.id, l.clone());
+                            }
                             Row::PlaylistTreeNode(n) => {
                                 nodes.insert(n.id, n.clone());
                             }
@@ -63,71 +230,6 @@ impl Database {
             v.sort_by_key(|e| e.entry_index); // needs `pub entry_index`
         }
 
-        Ok(Self { header, tracks, nodes, entries })
-    }
-    /// Export to low-level tables.
-    pub fn into_tables(
-        self,
-    ) -> (
-        Header,
-        Vec<Track>,
-        Vec<PlaylistTreeNode>,
-        Vec<(PlaylistTreeNodeId, PlaylistEntry)>,
-    ) {
-        let nodes = self.nodes.into_values().collect();
-        let tracks = self.tracks.into_values().collect();
-        let mut flat_entries = Vec::new();
-        for (pid, vs) in self.entries {
-            for e in vs {
-                flat_entries.push((pid, e));
+        Ok(Self { header, tracks, albums, artists, genres, keys, labels, nodes, entries })
     }
 }
-        (self.header, tracks, nodes, flat_entries)
-    }
-
-    /// Add a ready-made entry (you build it).
-    pub fn add_entry(
-        &mut self,
-        playlist_id: PlaylistTreeNodeId,
-        entry: PlaylistEntry,
-        position: Option<usize>,
-    ) -> Result<(), DbError> {
-        if !self.nodes.contains_key(&playlist_id) {
-            return Err(DbError::UnknownPlaylist);
-        }
-        let vec = self.entries.entry(playlist_id).or_default();
-        match position {
-            Some(i) if i <= vec.len() => vec.insert(i, entry),
-            Some(_) => return Err(DbError::PositionOutOfBounds),
-            None => vec.push(entry),
-        }
-        Ok(())
-    }
-
-    /// Convenience: you pass a track id and a constructor for `PlaylistEntry`.
-    pub fn add_track_to_playlist_with<E>(
-        &mut self,
-        playlist_id: PlaylistTreeNodeId,
-        track_id: TrackId,
-        position: Option<usize>,
-        make_entry: E,
-    ) -> Result<(), DbError>
-    where
-        E: FnOnce(TrackId) -> PlaylistEntry,
-    {
-        if !self.nodes.contains_key(&playlist_id) {
-            return Err(DbError::UnknownPlaylist);
-        }
-        if !self.tracks.contains_key(&track_id) {
-            return Err(DbError::UnknownTrack);
-        }
-        let entry = make_entry(track_id);
-        self.add_entry(playlist_id, entry, position)
-    }
-
-    // fn alloc_node_id(&mut self) -> u32 {
-    //     let id = self.next_node_id;
-    //     self.next_node_id = self.next_node_id.saturating_add(1);
-    //     id
-    // }
-}

maple-rs/src/main.rs

@@ -13,6 +13,8 @@ use rekordcrate::pdb::Table;
 use rekordcrate::pdb::{Header, PlaylistTreeNode, PlaylistTreeNodeId};
 use walkdir::WalkDir;
 use std::env;
+use std::fs::create_dir;
+use std::fs::create_dir_all;
 use std::{fs::exists, path::Path};
 use db::Database;
 
@@ -37,6 +39,8 @@ fn main() -> rekordcrate::Result<()> {
     //     None => println!("wasnt there"),
     // }
 
+    create_dir_all("./data/PIONEER")?;
+
     assert!(env::set_current_dir(&Path::new("./data")).is_ok());
 
     // strip the leading slash of the path to make it relative
@@ -102,6 +106,8 @@ fn main() -> rekordcrate::Result<()> {
 
     println!("export={:?}", export);
 
+    db.header.tables.iter().for_each(|t| println!("{:?}", t.page_type));
+
     // write header
     // write page
     // write row_groups