use std::mem::size_of; use zerocopy::{AsBytes, FromBytes, FromZeroes, Unaligned}; use crate::{Db, FilePointer, FileRange, PagePointer, RawFilePointer, PAGE_SIZE, U16, U32, U64}; #[derive(Clone, Copy, PartialEq, Eq, Debug)] enum SlabKind { SingleBytes, RelativeFreeList, AbsoluteFreeList, } impl SlabKind { fn for_size(size: u32) -> Self { if size == 1 { Self::SingleBytes } else if size < size_of::() as u32 { Self::RelativeFreeList } else if (size as u64) <= PAGE_SIZE / 2 { // TODO // slabs of really big objects are very inefficient. // find a better way/ allocate e.g. more pages at once or at least 10 elements? Self::AbsoluteFreeList } else { panic!("invalid size") } } } #[derive(Clone, Copy, FromBytes, FromZeroes, AsBytes, Unaligned)] #[repr(C)] pub struct AllocatorState { pub general: RawFilePointer, pub slabs: SlabListPointer, } #[derive(Clone, Copy, FromBytes, FromZeroes, AsBytes, Unaligned)] #[repr(transparent)] pub struct GeneralPurposeAllocator { pub head_ptr: FilePointer, } #[derive(Clone, Copy, FromBytes, FromZeroes, AsBytes, Unaligned)] #[repr(C)] pub struct FreeListBlock { next: FilePointer, size: u8, } impl FilePointer { pub fn next_ptr(self) -> FilePointer> { FilePointer::new(self.into_raw()) } pub fn size_start_ptr(self) -> FilePointer { FilePointer::new(self.into_raw() + size_of::>() as u64) } pub fn size_end_ptr(self) -> FilePointer { FilePointer::new(self.into_raw() + size_of::() as u64) } } impl GeneralPurposeAllocator { const SIZE_MASK: u8 = 0b1000_0000; const MIN_ALLOCATION_SIZE: u64 = size_of::() as u64; pub fn size(db: &Db, head: FilePointer) -> u64 { // println!("get size({head:?})"); let first_byte: u8 = unsafe { db.read(head.size_start_ptr()) }; let size = if first_byte & Self::SIZE_MASK == 0 { // small size (can fit in 7bits) first_byte as u64 } else { // large size unsafe { db.read::(head.size_end_ptr()) }.get() }; Self::MIN_ALLOCATION_SIZE + size } fn set_size(db: &mut Db, head: FilePointer, size: u64) { assert!(size >= Self::MIN_ALLOCATION_SIZE); let size = size - Self::MIN_ALLOCATION_SIZE; if size <= (u8::MAX & !Self::SIZE_MASK) as u64 { // small size (can fit in 7bits) debug_assert_eq!(size as u8 & Self::SIZE_MASK, 0); unsafe { db.write(head.size_start_ptr(), size as u8) }; } else { unsafe { db.write(head.size_start_ptr(), Self::SIZE_MASK); db.write::(head.size_end_ptr(), size.into()); } } } fn clear(db: &mut Db, ptr: FilePointer) -> RawFilePointer { unsafe { db.write(ptr.next_ptr(), FilePointer::null()); let first_byte: u8 = db.read(ptr.size_start_ptr()); // clear first size byte db.write(ptr.size_start_ptr(), 0); if first_byte & Self::SIZE_MASK != 0 { // larger block. clear full size field db.write(ptr.size_end_ptr(), 0.into()); } } ptr.into_raw() } fn can_allocate_into(needed_size: u64, actual_size: u64) -> bool { use std::cmp::Ordering::*; match actual_size.cmp(&needed_size) { Less => false, Equal => true, // leave space to insert the remaining space into the free list Greater => actual_size >= needed_size + Self::MIN_ALLOCATION_SIZE, } } fn needed_pages(size: u64) -> u64 { let mut n_pages = div_round_up(size, PAGE_SIZE); let extra_space = n_pages * PAGE_SIZE - size; if extra_space != 0 && extra_space < Self::MIN_ALLOCATION_SIZE { // the extra space in the allocated pages is too small to // insert it into the free list. allocate an additional page. n_pages += 1; } n_pages } pub fn allocate(self, db: &mut Db, expected_size: u64) -> FileRange { // we need space to store the free list entry let needed_size = expected_size.max(Self::MIN_ALLOCATION_SIZE); let head = self.head_ptr; // if the first element is replaced update the head pointer let mut prevprev = FilePointer::::null(); let mut prev = head; let mut next: FilePointer = unsafe { db.read(head.next_ptr()) }; let empty_list = next.is_null(); while !next.is_null() && !Self::can_allocate_into(needed_size, Self::size(db, next)) { prevprev = prev; prev = next; next = unsafe { db.read(next) }.next; } // dbg!(next, Self::size(db, next)); let start = if next.is_null() { let (prev, start, prev_free) = if !empty_list { let prevlen = Self::size(db, prev); if prev.into_raw() + prevlen == db.end_of_file() { // println!("free block at end of file {prev:?}"); Self::clear(db, prev); (prevprev, prev, prevlen) } else { (prev, FilePointer::new(db.end_of_file()), 0) } } else { (prev, FilePointer::new(db.end_of_file()), 0) }; // dbg!(prev, start, prev_free); let still_needed = if prev_free > needed_size { assert!(needed_size + Self::MIN_ALLOCATION_SIZE > prev_free); needed_size + Self::MIN_ALLOCATION_SIZE - prev_free } else { needed_size - prev_free }; let n_pages = Self::needed_pages(still_needed); assert_ne!(n_pages, 0); let page_start = db.add_pages(n_pages).start(); // dbg!(n_pages, page_start); if prev_free == 0 { assert_eq!(page_start, start.into_raw()); } let free_space = prev_free + PAGE_SIZE * n_pages; let extra_space = free_space - needed_size; if extra_space != 0 { let remainder = FilePointer::::new(start.into_raw() + needed_size); Self::set_size(db, remainder, extra_space); // prev must be the current tail of the free list and the newly allocated space, being at the end of the file // must be the last element of the free list to keep it sorted. unsafe { db.write(prev.next_ptr(), remainder) }; } else { unsafe { db.write(prev.next_ptr(), FilePointer::::null()) }; } start } else { let start = next; let nextnext = unsafe { db.read(start.next_ptr()) }; let extra_space = Self::size(db, start) - needed_size; // dbg!(prev, nextnext, extra_space); if extra_space != 0 { let remainder = FilePointer::::new(start.into_raw() + needed_size); // dbg!(remainder); Self::set_size(db, remainder, extra_space); unsafe { db.write(prev.next_ptr(), remainder); db.write(remainder.next_ptr(), nextnext); } // println!("{:x?}", unsafe { db.read::<[u8; 9 + 8]>(remainder) }); } else { unsafe { db.write(prev.next_ptr(), nextnext) }; } start }; let start = Self::clear(db, start); start.range(expected_size) } pub fn free(self, db: &mut Db, range: FileRange) { // println!("free({range:?})"); let mut size = range.len().max(Self::MIN_ALLOCATION_SIZE); let mut start = FilePointer::::new(range.start); let head = self.head_ptr; let mut prevprev = FilePointer::null(); let mut prev = head; let mut next = unsafe { db.read(head.next_ptr()) }; while !next.is_null() && next < start { prevprev = prev; prev = next; next = unsafe { db.read(next.next_ptr()) }; } if start.into_raw() + size == next.into_raw() { // we can merge with the next range let nextlen = Self::size(db, next); let nextnext = unsafe { db.read(next.next_ptr()) }; // println!("merging with next range {:?}", next.range(nextlen)); Self::clear(db, next); next = nextnext; size += nextlen; } // we can't merge with the head pointer if prev != head && prev.into_raw() + Self::size(db, prev) == start.into_raw() { // we can merge with the previous range let prevlen = Self::size(db, prev); // println!("merging with previous range {:?}", prev.range(prevlen)); Self::clear(db, prev); start = prev; prev = prevprev; size += prevlen; } unsafe { db.write(prev.next_ptr(), start); db.write(start.next_ptr(), next); Self::set_size(db, start, size) } } } fn div_round_up(a: u64, b: u64) -> u64 { (a + b - 1) / b } #[derive(Clone, Copy, FromBytes, FromZeroes, AsBytes, Unaligned, Debug)] #[repr(C)] pub struct SlabListHeader { next: SlabListPointer, len: U32, size: U32, } #[derive(Clone, Copy, FromBytes, FromZeroes, AsBytes, Unaligned, Debug)] #[repr(transparent)] pub struct SlabListPointer(pub FilePointer); pub struct SlabListIterator<'db, R> { position: u32, db: &'db Db, ptr: SlabListPointer, } impl<'db, R> Iterator for SlabListIterator<'db, R> { type Item = SlabPointer; fn next(&mut self) -> Option { if let Some(res) = self.ptr.get(self.db, self.position) { self.position += 1; Some(res) } else if let Some(next) = self.ptr.next(self.db) { self.ptr = next; self.next() } else { None } } } impl SlabListHeader { pub fn capacity(&self) -> u32 { (self.size.get() - size_of::() as u32) / size_of::() as u32 } } impl SlabListPointer { pub fn next(self, db: &Db) -> Option { let ptr: SlabListPointer = self.read_header(db).next; (!ptr.0.is_null()).then_some(ptr) } fn read_header(self, db: &Db) -> SlabListHeader { unsafe { db.read(self.0) } } fn modify_header(self, db: &mut Db) -> &mut SlabListHeader { unsafe { db.modify(self.0) } } pub fn set_next(self, db: &mut Db, next: SlabListPointer) { self.modify_header(db).next = next; } pub fn set_len(self, db: &mut Db, len: u32) { self.modify_header(db).len = U32::from(len); } pub fn init(self, db: &mut Db, size: u32) { *self.modify_header(db) = SlabListHeader { next: SlabListPointer(FilePointer::null()), size: size.into(), len: 0.into(), }; } pub fn element_ptr(self, db: &Db, i: u32) -> Option> { let this = self.read_header(db); (i < this.len.get()).then(|| { FilePointer::new( self.0.into_raw() + size_of::() as u64 + i as u64 * size_of::() as u64, ) }) } pub fn write_element(self, db: &mut Db, i: u32, value: Slab) { let ptr = self.element_ptr(db, i).unwrap(); unsafe { db.write(ptr, value) }; } pub fn get(self, db: &Db, i: u32) -> Option { self.element_ptr(db, i).map(SlabPointer) } pub fn iter(self, db: &Db) -> SlabListIterator { SlabListIterator { position: 0, db, ptr: self, } } pub fn add_slab(self, db: &mut Db, slab_size: u32) -> SlabPointer { println!("add_slab({slab_size})"); let this = self.read_header(db); dbg!(&this); let capacity = this.capacity(); let SlabListHeader { mut next, len, .. } = this; if len.get() >= capacity { if next.0.is_null() { next = SlabListPointer(FilePointer::new(db.add_pages(1).start())); next.init(db, PAGE_SIZE as u32); self.set_next(db, next); } return next.add_slab(db, slab_size); } let len = len.get(); self.set_len(db, len + 1); self.write_element( db, len, Slab { head: RawFilePointer::null(), size: slab_size.into(), }, ); SlabPointer(self.element_ptr(db, len).unwrap()) } } #[derive(Clone, Copy, FromBytes, FromZeroes, AsBytes, Unaligned)] #[repr(C)] pub struct Slab { head: RawFilePointer, size: U32, } #[derive(Clone, Copy, FromBytes, FromZeroes, AsBytes, Unaligned)] #[repr(C)] struct RelativeFreeListHeader { next_page: PagePointer, first: U16, } impl RelativeFreeListHeader { const DATA_SIZE: u32 = PAGE_SIZE as u32 - size_of::() as u32; fn capacity(size: u32) -> u32 { debug_assert_eq!(SlabKind::for_size(size), SlabKind::RelativeFreeList); Self::DATA_SIZE / size } } #[derive(Clone, Copy, FromBytes, FromZeroes, AsBytes, Unaligned)] #[repr(transparent)] pub struct SlabPointer(FilePointer); impl SlabPointer { fn read(&self, db: &Db) -> Slab { unsafe { db.read(self.0) } } fn modify<'db, R>(&self, db: &'db mut Db) -> &'db mut Slab { unsafe { db.modify(self.0) } } pub fn size(&self, db: &Db) -> u32 { self.read(db).size.get() } pub fn alloc(&self, db: &mut Db) -> FileRange { let Slab { mut head, size } = self.read(db); if head.is_null() { head = self.allocate_page(db); } let size = size.get(); match SlabKind::for_size(size) { SlabKind::SingleBytes => todo!("single byte slabs"), SlabKind::RelativeFreeList => { let (page, offset) = head.page_offset(); let start = FilePointer::::new(page.start()); assert_eq!(offset, 0); let RelativeFreeListHeader { first, .. } = unsafe { db.read(start) }; // the page should never be full if its in the free list assert_ne!(first.get(), 0); let ptr = FilePointer::::new(RawFilePointer::from_page_and_offset( page, first.get(), )); let next: U16 = unsafe { db.read(ptr) }; let header = unsafe { db.modify::(start) }; header.first = next; if next.get() == 0 { // page is full let next_page = header.next_page; header.next_page = PagePointer::null(); self.modify(db).head = next_page.start(); } ptr.into_raw() } SlabKind::AbsoluteFreeList => { let next = unsafe { db.read(FilePointer::::new(head)) }; self.set_head(db, next); head } } .range(size as u64) } pub fn free(&self, db: &mut Db, range: FileRange) { let Slab { head, size } = self.read(db); assert_eq!(range.len(), size.get() as u64); let size = size.get(); match SlabKind::for_size(size) { SlabKind::SingleBytes => todo!("single byte slabs"), SlabKind::RelativeFreeList => { let (page, offset) = range.start.page_offset(); let start = FilePointer::::new(page.start()); let RelativeFreeListHeader { first, .. } = unsafe { db.read(start) }; // update next pointer of new element in free list unsafe { db.write(FilePointer::::new(range.start), first) }; let header = unsafe { db.modify::(start) }; // point to new element header.first = offset.into(); if first.get() == 0 { // page was full let (head_page, offset) = head.page_offset(); assert_eq!(offset, 0); header.next_page = head_page; self.modify(db).head = page.start(); } } SlabKind::AbsoluteFreeList => { unsafe { db.write(FilePointer::::new(range.start), head) }; self.set_head(db, range.start); } } } pub fn set_head(&self, db: &mut Db, next: RawFilePointer) { self.modify(db).head = next; } pub fn allocate_page(&self, db: &mut Db) -> RawFilePointer { let Slab { head, size } = self.read(db); println!("allocate_slab_page({size})"); let size = size.get(); match SlabKind::for_size(size) { SlabKind::SingleBytes => todo!("single byte slabs"), SlabKind::RelativeFreeList => { let page = db.add_pages(1); let (next_page, offset) = head.page_offset(); assert_eq!(offset, 0); let capacity = RelativeFreeListHeader::capacity(size); let data_offset = size_of::() as u16; unsafe { db.write( FilePointer::new(page.start()), RelativeFreeListHeader { next_page, first: data_offset.into(), }, ) }; let mut offset = 0; for i in (0..capacity).rev() { let next = data_offset + (i * size) as u16; unsafe { db.write( FilePointer::new(RawFilePointer::from_page_and_offset(page, next)), U16::from(offset), ) }; offset = next; } self.set_head(db, page.start()); page.start() } SlabKind::AbsoluteFreeList => { let n = PAGE_SIZE / size as u64; let page = db.add_pages(1); let mut next = head; for i in (0..n).rev() { let current = page.start() + i * size as u64; unsafe { db.write(FilePointer::::new(current), next) }; next = current; } self.set_head(db, next); next } } } }