diff --git a/maple-rs/Cargo.toml b/maple-rs/Cargo.toml index e5eb6eb..c8652ad 100644 --- a/maple-rs/Cargo.toml +++ b/maple-rs/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] # rekordcrate = { git = "ssh://git@github.com/ntwl-xyz/rekordcrate.git", branch = "db" } -rekordcrate = { path = "../../rekordcrate" } +rekordcrate = { path = "../../code/rekordcrate" } lofty = "0.21" walkdir = "2.4" binrw = "0.15" diff --git a/maple-rs/src/db.rs b/maple-rs/src/db.rs index 113265c..73a6d8e 100644 --- a/maple-rs/src/db.rs +++ b/maple-rs/src/db.rs @@ -1,10 +1,30 @@ +use rekordcrate::pdb::Artwork; +use rekordcrate::pdb::ArtworkId; +use rekordcrate::pdb::Page; +use rekordcrate::pdb::PageType; +use rekordcrate::pdb::PageIndex; +use std::io::Write; +use std::io::{Seek, SeekFrom}; +use binrw::BinWrite; +use rekordcrate::pdb::LabelId; +use rekordcrate::pdb::KeyId; +use rekordcrate::pdb::GenreId; +use rekordcrate::pdb::ArtistId; +use rekordcrate::pdb::AlbumId; +use rekordcrate::pdb::Album; +use rekordcrate::pdb::Artist; +use rekordcrate::pdb::Genre; +use rekordcrate::pdb::Key; +use rekordcrate::pdb::Label; +use rekordcrate::pdb::Table; +use std::fs::File; use std::path::Path; use binrw::BinRead; use std::collections::BTreeMap; use binrw::Endian; use rekordcrate::pdb::{ - Header, PageType, Row, + Header, Row, Track, TrackId, PlaylistTreeNode, PlaylistTreeNodeId, PlaylistEntry, @@ -21,17 +41,149 @@ pub enum DbError { pub struct Database { pub header: Header, pub tracks: BTreeMap, + pub albums: BTreeMap, + pub artists: BTreeMap, + pub artwork: BTreeMap, + pub genres: BTreeMap, + pub keys: BTreeMap, + pub labels: BTreeMap, pub nodes: BTreeMap, // Keep playlist entries ordered by entry_index pub entries: BTreeMap>, } +const PAGE_SIZE: u32 = 4096; + impl Database { + pub fn export(self, path: &Path) -> std::io::Result<()> { + // 1) Reserve header page + let mut f = File::create(path)?; + f.write_all(&vec![0u8; PAGE_SIZE as usize])?; + + // 2) Emit tables linearly, collect table metadata + let mut next_page_index: u32 = 1; + + // ok now we're getting somewhere + // smash rows into row_groups of 16 + // try to write those into a page until it is full + // yield pages until all the rows are in + // set the indices + // 420 blaze it + + let (tracks_first, tracks_last) = + Self::write_table(&mut f, PageType::Tracks, next_page_index, self.tracks.values().cloned().map(Row::Track))?; + next_page_index = tracks_last + 1; + + let (tree_first, tree_last) = + Self::write_table(&mut f, PageType::PlaylistTree, next_page_index, self.nodes.values().cloned().map(Row::PlaylistTreeNode))?; + next_page_index = tree_last + 1; + + let plist_entries = self.entries.iter().flat_map(|(_pid, vec)| { + // ensure contiguous entry_index + let mut v = vec.clone(); + v.sort_by_key(|e| e.entry_index); + v.into_iter().map(Row::PlaylistEntry) + }); + let (entries_first, entries_last) = + Self::write_table(&mut f, PageType::PlaylistEntries, next_page_index, plist_entries)?; + next_page_index = entries_last + 1; + + let (art_first, art_last) = + Self::write_table(&mut f, PageType::Artwork, next_page_index, self.artwork.values().cloned().map(Row::Artwork))?; + next_page_index = art_last + 1; + + // 3) Build header and back-patch at page 0 + let header = Header { + // fill other required fields in your crate + sequence: 1, // or previous+1 + next_unused_page: PageIndex(next_page_index), + tables: vec![ + Table { page_type: PageType::Tracks, first_page: PageIndex(tracks_first), last_page: PageIndex(tracks_last), empty_candidate: 0, /* … */ }, + Table { page_type: PageType::PlaylistTree, first_page: PageIndex(tree_first), last_page: PageIndex(tree_last), empty_candidate: 0, /* … */ }, + Table { page_type: PageType::PlaylistEntries, first_page: PageIndex(entries_first), last_page: PageIndex(entries_last), empty_candidate: 0, /* … */ }, + Table { page_type: PageType::Artwork, first_page: PageIndex(art_first), last_page: PageIndex(art_last), empty_candidate: 0, /* … */ }, + ], + ..Default::default() + }; + + // Write header at offset 0 and pad to one page + f.seek(SeekFrom::Start(0))?; + header.write(&mut f); + // pad residual of page 0 if BinWrite didn’t fill it: + let pos = f.stream_position()?; + if pos < PAGE_SIZE as u64 { + f.write_all(&vec![0u8; (PAGE_SIZE as u64 - pos) as usize])?; + } + Ok(()) + } + + fn write_table( + f: &mut File, + page_type: PageType, + start_index: u32, + rows: I, + ) -> std::io::Result<(u32, u32)> + where + I: IntoIterator, + { + let mut page_index = start_index; + let mut first = 0; + let mut last = 0; + + // Pack rows into pages. Use your crate’s Page builder to handle row groups. + let mut packer = PagePacker::new(page_type, PAGE_SIZE); + + for row in rows { + if !packer.try_push(&row) { + // flush full page + let mut page = packer.finish(page_index, /*next_page=*/0); + // Temporarily set next_page=0; we will fix it after writing the next page. + Self::write_page(f, &mut page)?; + if first == 0 { first = page_index; } + last = page_index; + page_index += 1; + // start new page with the row that didn’t fit + packer = PagePacker::new(page_type, PAGE_SIZE); + assert!(packer.try_push(&row)); + } + } + + // flush final page if it has content or if this is an empty table create one empty page + if packer.is_empty() && first == 0 { + let mut page = Page::empty(page_type, page_index, /*next_page=*/0, PAGE_SIZE); + Self::write_page(f, &mut page)?; + first = page_index; + last = page_index; + } else { + let mut page = packer.finish(page_index, /*next_page=*/0); + Self::write_page(f, &mut page)?; + if first == 0 { first = page_index; } + last = page_index; + } + + // Fix `next_page` pointers by rewriting page headers if your Page writer requires it. + // Simpler: during emission, remember the previous page file offset and seek back + // to patch its `next_page` to `page_index`. + + Ok((first, last)) + } + + fn write_page(f: &mut File, page: &mut Page) -> std::io::Result<()> { + page.write(f); + Ok(()) + } + pub fn load_from_pdb(path: impl AsRef) -> rekordcrate::Result { let mut f = std::fs::File::open(path)?; let header = Header::read(&mut f)?; // uses the type’s endianness let mut tracks: BTreeMap = BTreeMap::new(); + let mut albums: BTreeMap = BTreeMap::new(); + let mut artists: BTreeMap = BTreeMap::new(); + let mut genres: BTreeMap = BTreeMap::new(); + let mut keys: BTreeMap = BTreeMap::new(); + let mut labels: BTreeMap = BTreeMap::new(); + let mut nodes: BTreeMap = BTreeMap::new(); let mut entries: BTreeMap> = BTreeMap::new(); @@ -45,6 +197,21 @@ impl Database { Row::Track(t) => { tracks.insert(t.id, t.clone()); } + Row::Artist(a) => { + artists.insert(a.id, a.clone()); + } + Row::Album(a) => { + albums.insert(a.id, a.clone()); + } + Row::Genre(g) => { + genres.insert(g.id, g.clone()); + } + Row::Key(k) => { + keys.insert(k.id, k.clone()); + } + Row::Label(l) => { + labels.insert(l.id, l.clone()); + } Row::PlaylistTreeNode(n) => { nodes.insert(n.id, n.clone()); } @@ -63,71 +230,6 @@ impl Database { v.sort_by_key(|e| e.entry_index); // needs `pub entry_index` } - Ok(Self { header, tracks, nodes, entries }) + Ok(Self { header, tracks, albums, artists, genres, keys, labels, nodes, entries }) } - /// Export to low-level tables. - pub fn into_tables( - self, - ) -> ( - Header, - Vec, - Vec, - Vec<(PlaylistTreeNodeId, PlaylistEntry)>, - ) { - let nodes = self.nodes.into_values().collect(); - let tracks = self.tracks.into_values().collect(); - let mut flat_entries = Vec::new(); - for (pid, vs) in self.entries { - for e in vs { - flat_entries.push((pid, e)); - } - } - (self.header, tracks, nodes, flat_entries) - } - - /// Add a ready-made entry (you build it). - pub fn add_entry( - &mut self, - playlist_id: PlaylistTreeNodeId, - entry: PlaylistEntry, - position: Option, - ) -> Result<(), DbError> { - if !self.nodes.contains_key(&playlist_id) { - return Err(DbError::UnknownPlaylist); - } - let vec = self.entries.entry(playlist_id).or_default(); - match position { - Some(i) if i <= vec.len() => vec.insert(i, entry), - Some(_) => return Err(DbError::PositionOutOfBounds), - None => vec.push(entry), - } - Ok(()) - } - - /// Convenience: you pass a track id and a constructor for `PlaylistEntry`. - pub fn add_track_to_playlist_with( - &mut self, - playlist_id: PlaylistTreeNodeId, - track_id: TrackId, - position: Option, - make_entry: E, - ) -> Result<(), DbError> - where - E: FnOnce(TrackId) -> PlaylistEntry, - { - if !self.nodes.contains_key(&playlist_id) { - return Err(DbError::UnknownPlaylist); - } - if !self.tracks.contains_key(&track_id) { - return Err(DbError::UnknownTrack); - } - let entry = make_entry(track_id); - self.add_entry(playlist_id, entry, position) - } - - // fn alloc_node_id(&mut self) -> u32 { - // let id = self.next_node_id; - // self.next_node_id = self.next_node_id.saturating_add(1); - // id - // } } diff --git a/maple-rs/src/main.rs b/maple-rs/src/main.rs index 9f4a31b..58688d8 100644 --- a/maple-rs/src/main.rs +++ b/maple-rs/src/main.rs @@ -13,6 +13,8 @@ use rekordcrate::pdb::Table; use rekordcrate::pdb::{Header, PlaylistTreeNode, PlaylistTreeNodeId}; use walkdir::WalkDir; use std::env; +use std::fs::create_dir; +use std::fs::create_dir_all; use std::{fs::exists, path::Path}; use db::Database; @@ -37,6 +39,8 @@ fn main() -> rekordcrate::Result<()> { // None => println!("wasnt there"), // } + create_dir_all("./data/PIONEER")?; + assert!(env::set_current_dir(&Path::new("./data")).is_ok()); // strip the leading slash of the path to make it relative @@ -102,6 +106,8 @@ fn main() -> rekordcrate::Result<()> { println!("export={:?}", export); + db.header.tables.iter().for_each(|t| println!("{:?}", t.page_type)); + // write header // write page // write row_groups