Initial commit

mdb/pfile/alloclist.go

@@ -0,0 +1,57 @@
+package pfile
+
+import "slices"
+
+type allocList map[[2]uint64][]uint64
+
+func newAllocList() *allocList {
+	al := allocList(map[[2]uint64][]uint64{})
+	return &al
+}
+
+func (al allocList) Create(collectionID, itemID, page uint64) {
+	key := al.key(collectionID, itemID)
+	al[key] = []uint64{page}
+}
+
+// Push is used to add pages to the storage when loading. It will append
+// pages to the appropriate list, or return false if the list isn't found.
+func (al allocList) Push(collectionID, itemID, page uint64) bool {
+	key := al.key(collectionID, itemID)
+	if _, ok := al[key]; !ok {
+		return false
+	}
+	al[key] = append(al[key], page)
+	return true
+}
+
+func (al allocList) Store(collectionID, itemID uint64, pages []uint64) {
+	key := al.key(collectionID, itemID)
+	al[key] = slices.Clone(pages)
+}
+
+func (al allocList) Remove(collectionID, itemID uint64) []uint64 {
+	key := al.key(collectionID, itemID)
+	pages := al[key]
+	delete(al, key)
+	return pages
+}
+
+func (al allocList) Iterate(
+	each func(collectionID, itemID uint64, pages []uint64) error,
+) error {
+	for key, pages := range al {
+		if err := each(key[0], key[1], pages); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (al allocList) Len() int {
+	return len(al)
+}
+
+func (al allocList) key(collectionID, itemID uint64) [2]uint64 {
+	return [2]uint64{collectionID, itemID}
+}

mdb/pfile/alloclist_test.go

@@ -0,0 +1,172 @@
+package pfile
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+)
+
+func (al allocList) Assert(t *testing.T, state map[[2]uint64][]uint64) {
+	t.Helper()
+
+	if len(al) != len(state) {
+		t.Fatalf("Expected %d items, but found %d.", len(state), len(al))
+	}
+
+	for key, expected := range state {
+		val, ok := al[key]
+		if !ok {
+			t.Fatalf("Expected to find key %v.", key)
+		}
+		if !reflect.DeepEqual(val, expected) {
+			t.Fatalf("For %v, expected %v but got %v.", key, expected, val)
+		}
+	}
+}
+
+func (al *allocList) With(collectionID, itemID uint64, pages ...uint64) *allocList {
+	al.Store(collectionID, itemID, pages)
+	return al
+}
+
+func (al *allocList) Equals(rhs *allocList) bool {
+
+	if len(*rhs) != len(*al) {
+		return false
+	}
+
+	for key, val := range *rhs {
+		actual := (*al)[key]
+		if !reflect.DeepEqual(val, actual) {
+			return false
+		}
+	}
+	return true
+}
+
+func TestAllocList(t *testing.T) {
+	const (
+		CREATE = "CREATE"
+		PUSH   = "PUSH"
+		STORE  = "STORE"
+		REMOVE = "REMOVE"
+	)
+
+	type TestCase struct {
+		Name        string
+		Action      string
+		Key         [2]uint64
+		Page        uint64
+		Pages       []uint64 // For STORE command.
+		Expected    *allocList
+		ExpectedLen int
+	}
+
+	testCases := []TestCase{{
+		Name:        "Create something",
+		Action:      CREATE,
+		Key:         [2]uint64{1, 1},
+		Page:        1,
+		Expected:    newAllocList().With(1, 1, 1),
+		ExpectedLen: 1,
+	}, {
+		Name:        "Push onto something",
+		Action:      PUSH,
+		Key:         [2]uint64{1, 1},
+		Page:        2,
+		Expected:    newAllocList().With(1, 1, 1, 2),
+		ExpectedLen: 1,
+	}, {
+		Name:        "Push onto something again",
+		Action:      PUSH,
+		Key:         [2]uint64{1, 1},
+		Page:        3,
+		Expected:    newAllocList().With(1, 1, 1, 2, 3),
+		ExpectedLen: 1,
+	}, {
+		Name:        "Store something",
+		Action:      STORE,
+		Key:         [2]uint64{2, 2},
+		Pages:       []uint64{4, 5, 6},
+		Expected:    newAllocList().With(1, 1, 1, 2, 3).With(2, 2, 4, 5, 6),
+		ExpectedLen: 2,
+	}, {
+		Name:        "Remove something",
+		Action:      REMOVE,
+		Key:         [2]uint64{1, 1},
+		Expected:    newAllocList().With(2, 2, 4, 5, 6),
+		ExpectedLen: 1,
+	}}
+
+	al := newAllocList()
+
+	for _, tc := range testCases {
+		switch tc.Action {
+		case CREATE:
+			al.Create(tc.Key[0], tc.Key[1], tc.Page)
+
+		case PUSH:
+			al.Push(tc.Key[0], tc.Key[1], tc.Page)
+
+		case STORE:
+			al.Store(tc.Key[0], tc.Key[1], tc.Pages)
+
+		case REMOVE:
+			al.Remove(tc.Key[0], tc.Key[1])
+
+		default:
+			t.Fatalf("Unknown action: %s", tc.Action)
+		}
+
+		if !al.Equals(tc.Expected) {
+			t.Fatal(tc.Name, al, tc.Expected)
+		}
+
+		if al.Len() != tc.ExpectedLen {
+			t.Fatal(tc.Name, al.Len(), tc.ExpectedLen)
+		}
+	}
+}
+
+func TestAllocListIterate_eachError(t *testing.T) {
+	al := newAllocList().With(1, 1, 2, 3, 4, 5)
+	myErr := errors.New("xxx")
+	err := al.Iterate(func(collectionID, itemID uint64, pageIDs []uint64) error {
+		return myErr
+	})
+	if err != myErr {
+		t.Fatal(err)
+	}
+}
+
+func TestAllocListIterate(t *testing.T) {
+	al := newAllocList().With(1, 1, 2, 3, 4, 5).With(2, 2, 6, 7)
+	expected := map[uint64][]uint64{
+		1: {2, 3, 4, 5},
+		2: {6, 7},
+	}
+
+	err := al.Iterate(func(collectionID, itemID uint64, pageIDs []uint64) error {
+		e, ok := expected[collectionID]
+		if !ok {
+			t.Fatalf("Not found: %d", collectionID)
+		}
+		if !reflect.DeepEqual(e, pageIDs) {
+			t.Fatalf("%v != %v", pageIDs, e)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestAllocListPushNoHead(t *testing.T) {
+	al := newAllocList().With(1, 1, 2, 3, 4, 5).With(2, 2, 6, 7)
+	if !al.Push(1, 1, 8) {
+		t.Fatal("Failed to push onto head page")
+	}
+	if al.Push(1, 2, 9) {
+		t.Fatal("Pushed with no head.")
+	}
+}

mdb/pfile/change_test.go

@@ -0,0 +1,58 @@
+package pfile
+
+import (
+	crand "crypto/rand"
+	"git.crumpington.com/public/jldb/mdb/change"
+	"math/rand"
+)
+
+func randomChangeList() (changes []change.Change) {
+	count := 1 + rand.Intn(8)
+	for i := 0; i < count; i++ {
+		change := change.Change{
+			CollectionID: 1 + uint64(rand.Int63n(10)),
+			ItemID:       1 + uint64(rand.Int63n(10)),
+		}
+
+		if rand.Float32() < 0.95 {
+			change.Data = randBytes(1 + rand.Intn(pageDataSize*4))
+			change.Store = true
+		}
+
+		changes = append(changes, change)
+	}
+
+	return changes
+}
+
+type changeListBuilder []change.Change
+
+func (b *changeListBuilder) Clear() *changeListBuilder {
+	*b = (*b)[:0]
+	return b
+}
+
+func (b *changeListBuilder) Store(cID, iID, dataSize uint64) *changeListBuilder {
+	data := make([]byte, dataSize)
+	crand.Read(data)
+	*b = append(*b, change.Change{
+		CollectionID: cID,
+		ItemID:       iID,
+		Store:        true,
+		Data:         data,
+	})
+	return b
+}
+
+func (b *changeListBuilder) Delete(cID, iID uint64) *changeListBuilder {
+	*b = append(*b, change.Change{
+		CollectionID: cID,
+		ItemID:       iID,
+		Store:        false,
+	})
+	return b
+}
+
+func (b *changeListBuilder) Build() []change.Change {
+	return *b
+}

mdb/pfile/freelist.go

@@ -0,0 +1,67 @@
+package pfile
+
+import "container/heap"
+
+// ----------------------------------------------------------------------------
+// The intHeap is used to store the free list.
+// ----------------------------------------------------------------------------
+
+type intHeap []uint64
+
+func (h intHeap) Len() int           { return len(h) }
+func (h intHeap) Less(i, j int) bool { return h[i] < h[j] }
+func (h intHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+
+func (h *intHeap) Push(x any) {
+	// Push and Pop use pointer receivers because they modify the slice's length,
+	// not just its contents.
+	*h = append(*h, x.(uint64))
+}
+
+func (h *intHeap) Pop() any {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[0 : n-1]
+	return x
+}
+
+// ----------------------------------------------------------------------------
+// Free list
+// ----------------------------------------------------------------------------
+
+type freeList struct {
+	h        intHeap
+	nextPage uint64
+}
+
+// newFreeList creates a new free list that will return available pages from
+// smallest to largest. If there are no available pages, it will return new
+// pages starting from nextPage.
+func newFreeList(pageCount uint64) *freeList {
+	return &freeList{
+		h:        []uint64{},
+		nextPage: pageCount,
+	}
+}
+
+func (f *freeList) Push(pages ...uint64) {
+	for _, page := range pages {
+		heap.Push(&f.h, page)
+	}
+}
+
+func (f *freeList) Pop(count int, out []uint64) []uint64 {
+	out = out[:0]
+
+	for len(out) < count && len(f.h) > 0 {
+		out = append(out, heap.Pop(&f.h).(uint64))
+	}
+
+	for len(out) < count {
+		out = append(out, f.nextPage)
+		f.nextPage++
+	}
+
+	return out
+}

mdb/pfile/freelist_test.go

@@ -0,0 +1,90 @@
+package pfile
+
+import (
+	"math/rand"
+	"reflect"
+	"testing"
+)
+
+func (fl *freeList) Assert(t *testing.T, pageIDs ...uint64) {
+	t.Helper()
+
+	if len(fl.h) != len(pageIDs) {
+		t.Fatalf("FreeList: Expected %d pages but got %d.\n%v != %v",
+			len(pageIDs), len(fl.h), fl.h, pageIDs)
+	}
+
+	containsPageID := func(pageID uint64) bool {
+		for _, v := range fl.h {
+			if v == pageID {
+				return true
+			}
+		}
+		return false
+	}
+
+	for _, pageID := range pageIDs {
+		if !containsPageID(pageID) {
+			t.Fatalf("Page not free: %d", pageID)
+		}
+	}
+}
+
+func TestFreeList(t *testing.T) {
+	t.Parallel()
+
+	p0 := uint64(1 + rand.Int63())
+
+	type TestCase struct {
+		Name     string
+		Put      []uint64
+		Alloc    int
+		Expected []uint64
+	}
+
+	testCases := []TestCase{
+		{
+			Name:     "Alloc first page",
+			Put:      []uint64{},
+			Alloc:    1,
+			Expected: []uint64{p0},
+		}, {
+			Name:     "Alloc second page",
+			Put:      []uint64{},
+			Alloc:    1,
+			Expected: []uint64{p0 + 1},
+		}, {
+			Name:     "Put second page",
+			Put:      []uint64{p0 + 1},
+			Alloc:    0,
+			Expected: []uint64{},
+		}, {
+			Name:     "Alloc 2 pages",
+			Put:      []uint64{},
+			Alloc:    2,
+			Expected: []uint64{p0 + 1, p0 + 2},
+		}, {
+			Name:     "Put back and alloc pages",
+			Put:      []uint64{p0},
+			Alloc:    3,
+			Expected: []uint64{p0, p0 + 3, p0 + 4},
+		}, {
+			Name:     "Put back large and alloc",
+			Put:      []uint64{p0, p0 + 2, p0 + 4, p0 + 442},
+			Alloc:    4,
+			Expected: []uint64{p0, p0 + 2, p0 + 4, p0 + 442},
+		},
+	}
+
+	fl := newFreeList(p0)
+
+	var pages []uint64
+
+	for _, tc := range testCases {
+		fl.Push(tc.Put...)
+		pages = fl.Pop(tc.Alloc, pages)
+		if !reflect.DeepEqual(pages, tc.Expected) {
+			t.Fatal(tc.Name, pages, tc.Expected)
+		}
+	}
+}

mdb/pfile/header.go

@@ -0,0 +1 @@
+package pfile

mdb/pfile/index.go

@@ -0,0 +1,105 @@
+package pfile
+
+import (
+	"git.crumpington.com/public/jldb/lib/errs"
+	"git.crumpington.com/public/jldb/mdb/change"
+)
+
+type Index struct {
+	fList *freeList
+	aList allocList
+	seen  map[[2]uint64]struct{}
+	mask  []bool
+}
+
+func NewIndex(f *File) (*Index, error) {
+	idx := &Index{
+		fList: newFreeList(0),
+		aList: *newAllocList(),
+		seen:  map[[2]uint64]struct{}{},
+		mask:  []bool{},
+	}
+
+	err := f.iterate(func(pageID uint64, page dataPage) error {
+		header := page.Header()
+		switch header.PageType {
+		case pageTypeHead:
+			idx.aList.Create(header.CollectionID, header.ItemID, pageID)
+		case pageTypeData:
+			if !idx.aList.Push(header.CollectionID, header.ItemID, pageID) {
+				return errs.Corrupt.WithMsg("encountered data page with no corresponding head page")
+			}
+		case pageTypeFree:
+			idx.fList.Push(pageID)
+		}
+		return nil
+	})
+
+	return idx, err
+}
+
+func (idx *Index) StageChanges(changes []change.Change) {
+	clear(idx.seen)
+	if cap(idx.mask) < len(changes) {
+		idx.mask = make([]bool, len(changes))
+	}
+	idx.mask = idx.mask[:len(changes)]
+
+	for i := len(changes) - 1; i >= 0; i-- {
+		key := [2]uint64{changes[i].CollectionID, changes[i].ItemID}
+		if _, ok := idx.seen[key]; ok {
+			idx.mask[i] = false
+			continue
+		}
+
+		idx.seen[key] = struct{}{}
+		idx.mask[i] = true
+	}
+
+	for i, active := range idx.mask {
+		if !active {
+			continue
+		}
+
+		if changes[i].Store {
+			count := idx.getPageCountForData(len(changes[i].Data))
+			changes[i].WritePageIDs = idx.fList.Pop(count, changes[i].WritePageIDs)
+		}
+
+		if pages := idx.aList.Remove(changes[i].CollectionID, changes[i].ItemID); pages != nil {
+			changes[i].ClearPageIDs = pages
+		}
+	}
+}
+
+func (idx *Index) UnstageChanges(changes []change.Change) {
+	for i := range changes {
+		if len(changes[i].WritePageIDs) > 0 {
+			idx.fList.Push(changes[i].WritePageIDs...)
+			changes[i].WritePageIDs = changes[i].WritePageIDs[:0]
+		}
+		if len(changes[i].ClearPageIDs) > 0 {
+			idx.aList.Store(changes[i].CollectionID, changes[i].ItemID, changes[i].ClearPageIDs)
+			changes[i].ClearPageIDs = changes[i].ClearPageIDs[:0]
+		}
+	}
+}
+
+func (idx *Index) ApplyChanges(changes []change.Change) {
+	for i := range changes {
+		if len(changes[i].WritePageIDs) > 0 {
+			idx.aList.Store(changes[i].CollectionID, changes[i].ItemID, changes[i].WritePageIDs)
+		}
+		if len(changes[i].ClearPageIDs) > 0 {
+			idx.fList.Push(changes[i].ClearPageIDs...)
+		}
+	}
+}
+
+func (idx *Index) getPageCountForData(dataSize int) int {
+	count := dataSize / pageDataSize
+	if dataSize%pageDataSize != 0 {
+		count++
+	}
+	return count
+}

mdb/pfile/index_test.go

@@ -0,0 +1,139 @@
+package pfile
+
+import (
+	"testing"
+)
+
+type IndexState struct {
+	FreeList  []uint64
+	AllocList map[[2]uint64][]uint64
+}
+
+func (idx *Index) Assert(t *testing.T, state IndexState) {
+	t.Helper()
+
+	idx.fList.Assert(t, state.FreeList...)
+	idx.aList.Assert(t, state.AllocList)
+}
+
+func TestIndex(t *testing.T) {
+	pf, idx := newForTesting(t)
+	defer pf.Close()
+
+	idx.Assert(t, IndexState{
+		FreeList:  []uint64{},
+		AllocList: map[[2]uint64][]uint64{},
+	})
+
+	p0 := uint64(0)
+
+	l := (&changeListBuilder{}).
+		Store(1, 1, pageDataSize+1).
+		Build()
+
+	idx.StageChanges(l)
+
+	idx.Assert(t, IndexState{
+		FreeList:  []uint64{},
+		AllocList: map[[2]uint64][]uint64{},
+	})
+
+	// Unstage a change: free-list gets pages back.
+	idx.UnstageChanges(l)
+
+	idx.Assert(t, IndexState{
+		FreeList:  []uint64{p0, p0 + 1},
+		AllocList: map[[2]uint64][]uint64{},
+	})
+
+	// Stage a change: free-list entries are used again.
+	l = (*changeListBuilder)(&l).
+		Clear().
+		Store(1, 1, pageDataSize+1).
+		Store(2, 2, pageDataSize-1).
+		Store(3, 3, pageDataSize).
+		Build()
+
+	idx.StageChanges(l)
+
+	idx.Assert(t, IndexState{
+		FreeList:  []uint64{},
+		AllocList: map[[2]uint64][]uint64{},
+	})
+
+	// Apply changes: alloc-list is updated.
+	idx.ApplyChanges(l)
+
+	idx.Assert(t, IndexState{
+		FreeList: []uint64{},
+		AllocList: map[[2]uint64][]uint64{
+			{1, 1}: {p0, p0 + 1},
+			{2, 2}: {p0 + 2},
+			{3, 3}: {p0 + 3},
+		},
+	})
+
+	// Clear some things.
+	l = (*changeListBuilder)(&l).
+		Clear().
+		Store(1, 1, pageDataSize).
+		Delete(2, 2).
+		Build()
+
+	idx.StageChanges(l)
+
+	idx.Assert(t, IndexState{
+		FreeList: []uint64{},
+		AllocList: map[[2]uint64][]uint64{
+			{3, 3}: {p0 + 3},
+		},
+	})
+
+	// Ustaging will push the staged page p0+4 into the free list.
+	idx.UnstageChanges(l)
+
+	idx.Assert(t, IndexState{
+		FreeList: []uint64{p0 + 4},
+		AllocList: map[[2]uint64][]uint64{
+			{1, 1}: {p0, p0 + 1},
+			{2, 2}: {p0 + 2},
+			{3, 3}: {p0 + 3},
+		},
+	})
+
+	idx.StageChanges(l)
+
+	idx.Assert(t, IndexState{
+		FreeList: []uint64{},
+		AllocList: map[[2]uint64][]uint64{
+			{3, 3}: {p0 + 3},
+		},
+	})
+
+	idx.ApplyChanges(l)
+
+	idx.Assert(t, IndexState{
+		FreeList: []uint64{p0, p0 + 1, p0 + 2},
+		AllocList: map[[2]uint64][]uint64{
+			{1, 1}: {p0 + 4},
+			{3, 3}: {p0 + 3},
+		},
+	})
+
+	// Duplicate updates.
+	l = (*changeListBuilder)(&l).
+		Clear().
+		Store(2, 2, pageDataSize).
+		Store(3, 3, pageDataSize+1).
+		Store(3, 3, pageDataSize).
+		Build()
+
+	idx.StageChanges(l)
+
+	idx.Assert(t, IndexState{
+		FreeList: []uint64{p0 + 2},
+		AllocList: map[[2]uint64][]uint64{
+			{1, 1}: {p0 + 4},
+		},
+	})
+}

mdb/pfile/iterate.go

@@ -0,0 +1,18 @@
+package pfile
+
+import "bytes"
+
+func IterateAllocated(
+	pf *File,
+	idx *Index,
+	each func(collectionID, itemID uint64, data []byte) error,
+) error {
+	buf := &bytes.Buffer{}
+	return idx.aList.Iterate(func(collectionID, itemID uint64, pages []uint64) error {
+		buf.Reset()
+		if err := pf.readData(pages[0], buf); err != nil {
+			return err
+		}
+		return each(collectionID, itemID, buf.Bytes())
+	})
+}

mdb/pfile/main_test.go

@@ -0,0 +1,64 @@
+package pfile
+
+import (
+	"bytes"
+	crand "crypto/rand"
+	"git.crumpington.com/public/jldb/lib/wal"
+	"git.crumpington.com/public/jldb/mdb/change"
+	"path/filepath"
+	"testing"
+)
+
+func newForTesting(t *testing.T) (*File, *Index) {
+	t.Helper()
+
+	tmpDir := t.TempDir()
+	filePath := filepath.Join(tmpDir, "pagefile")
+
+	pf, err := Open(filePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	idx, err := NewIndex(pf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return pf, idx
+}
+
+func randBytes(size int) []byte {
+	buf := make([]byte, size)
+	if _, err := crand.Read(buf); err != nil {
+		panic(err)
+	}
+	return buf
+}
+
+func changesToRec(changes []change.Change) wal.Record {
+	buf := &bytes.Buffer{}
+	if err := change.Write(changes, buf); err != nil {
+		panic(err)
+	}
+	return wal.Record{
+		DataSize: int64(buf.Len()),
+		Reader:   buf,
+	}
+}
+
+func TestChangesToRec(t *testing.T) {
+	changes := []change.Change{
+		{
+			CollectionID: 2,
+			ItemID:       3,
+			Store:        true,
+			Data:         []byte{2, 3, 4},
+			WritePageIDs: []uint64{0, 1},
+			ClearPageIDs: []uint64{2, 3},
+		},
+	}
+	rec := changesToRec(changes)
+	c2 := []change.Change{}
+	c2, _ = change.Read(c2, rec.Reader)
+}

mdb/pfile/page.go

@@ -0,0 +1,70 @@
+package pfile
+
+import (
+	"hash/crc32"
+	"git.crumpington.com/public/jldb/lib/errs"
+	"unsafe"
+)
+
+// ----------------------------------------------------------------------------
+
+const (
+	pageSize       = 512
+	pageHeaderSize = 40
+	pageDataSize   = pageSize - pageHeaderSize
+
+	pageTypeFree = 0
+	pageTypeHead = 1
+	pageTypeData = 2
+)
+
+var emptyPage = func() dataPage {
+	p := newDataPage()
+	h := p.Header()
+	h.CRC = p.ComputeCRC()
+	return p
+}()
+
+// ----------------------------------------------------------------------------
+
+type pageHeader struct {
+	CRC          uint32 // IEEE CRC-32 checksum.
+	PageType     uint32 // One of the PageType* constants.
+	CollectionID uint64  //
+	ItemID       uint64
+	DataSize     uint64
+	NextPage     uint64
+}
+
+// ----------------------------------------------------------------------------
+
+type dataPage []byte
+
+func newDataPage() dataPage {
+	p := dataPage(make([]byte, pageSize))
+	return p
+}
+
+func (p dataPage) Header() *pageHeader {
+	return (*pageHeader)(unsafe.Pointer(&p[0]))
+}
+
+func (p dataPage) ComputeCRC() uint32 {
+	return crc32.ChecksumIEEE(p[4:])
+}
+
+func (p dataPage) Data() []byte {
+	return p[pageHeaderSize:]
+}
+
+func (p dataPage) Write(data []byte) int {
+	return copy(p[pageHeaderSize:], data)
+}
+
+func (p dataPage) Validate() error {
+	header := p.Header()
+	if header.CRC != p.ComputeCRC() {
+		return errs.Corrupt.WithMsg("CRC mismatch on data page.")
+	}
+	return nil
+}

mdb/pfile/page_test.go

@@ -0,0 +1,103 @@
+package pfile
+
+import (
+	"bytes"
+	crand "crypto/rand"
+	"git.crumpington.com/public/jldb/lib/errs"
+	"math/rand"
+	"testing"
+)
+
+func randomPage(t *testing.T) dataPage {
+	p := newDataPage()
+	h := p.Header()
+
+	x := rand.Float32()
+	if x > 0.66 {
+		h.PageType = pageTypeFree
+		h.DataSize = 0
+	} else if x < 0.33 {
+		h.PageType = pageTypeHead
+		h.DataSize = rand.Uint64()
+	} else {
+		h.PageType = pageTypeData
+		h.DataSize = rand.Uint64()
+	}
+
+	h.CollectionID = rand.Uint64()
+	h.ItemID = rand.Uint64()
+
+	dataSize := h.DataSize
+	if h.DataSize > pageDataSize {
+		dataSize = pageDataSize
+	}
+
+	if _, err := crand.Read(p.Data()[:dataSize]); err != nil {
+		t.Fatal(err)
+	}
+
+	h.CRC = p.ComputeCRC()
+
+	return p
+}
+
+// ----------------------------------------------------------------------------
+
+func TestPageValidate(t *testing.T) {
+	for i := 0; i < 100; i++ {
+		p := randomPage(t)
+
+		// Should be valid initially.
+		if err := p.Validate(); err != nil {
+			t.Fatal(err)
+		}
+
+		for i := 0; i < pageSize; i++ {
+			p[i]++
+			if err := p.Validate(); !errs.Corrupt.Is(err) {
+				t.Fatal(err)
+			}
+			p[i]--
+		}
+
+		// Should be valid initially.
+		if err := p.Validate(); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestPageEmptyIsValid(t *testing.T) {
+	if err := emptyPage.Validate(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestPageWrite(t *testing.T) {
+	for i := 0; i < 100; i++ {
+		page := newDataPage()
+		h := page.Header()
+		h.PageType = pageTypeData
+		h.CollectionID = rand.Uint64()
+		h.ItemID = rand.Uint64()
+		h.DataSize = uint64(1 + rand.Int63n(2*pageDataSize))
+
+		data := make([]byte, h.DataSize)
+		crand.Read(data)
+
+		n := page.Write(data)
+		h.CRC = page.ComputeCRC()
+
+		if n > pageDataSize || n < 1 {
+			t.Fatal(n)
+		}
+
+		if !bytes.Equal(data[:n], page.Data()[:n]) {
+			t.Fatal(data[:n], page.Data()[:n])
+		}
+
+		if err := page.Validate(); err != nil {
+			t.Fatal(err)
+		}
+	}
+}

mdb/pfile/pagefile.go

@@ -0,0 +1,307 @@
+package pfile
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"encoding/binary"
+	"io"
+	"git.crumpington.com/public/jldb/lib/errs"
+	"git.crumpington.com/public/jldb/mdb/change"
+	"net"
+	"os"
+	"sync"
+	"time"
+)
+
+type File struct {
+	lock sync.RWMutex
+	f    *os.File
+	page dataPage
+}
+
+func Open(path string) (*File, error) {
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600)
+	if err != nil {
+		return nil, errs.IO.WithErr(err)
+	}
+
+	pf := &File{f: f}
+	pf.page = newDataPage()
+
+	return pf, nil
+}
+
+func (pf *File) Close() error {
+	pf.lock.Lock()
+	defer pf.lock.Unlock()
+
+	if err := pf.f.Close(); err != nil {
+		return errs.IO.WithErr(err)
+	}
+	return nil
+}
+
+// ----------------------------------------------------------------------------
+// Writing
+// ----------------------------------------------------------------------------
+
+func (pf *File) ApplyChanges(changes []change.Change) error {
+	pf.lock.Lock()
+	defer pf.lock.Unlock()
+
+	return pf.applyChanges(changes)
+}
+
+func (pf *File) applyChanges(changes []change.Change) error {
+	for _, change := range changes {
+		if len(change.WritePageIDs) > 0 {
+			if err := pf.writeChangePages(change); err != nil {
+				return err
+			}
+		}
+
+		for _, id := range change.ClearPageIDs {
+			if err := pf.writePage(emptyPage, id); err != nil {
+				return err
+			}
+		}
+	}
+
+	if err := pf.f.Sync(); err != nil {
+		return errs.IO.WithErr(err)
+	}
+
+	return nil
+}
+
+func (pf *File) writeChangePages(change change.Change) error {
+	page := pf.page
+
+	header := page.Header()
+
+	header.PageType = pageTypeHead
+	header.CollectionID = change.CollectionID
+	header.ItemID = change.ItemID
+	header.DataSize = uint64(len(change.Data))
+
+	pageIDs := change.WritePageIDs
+	data := change.Data
+
+	for len(change.Data) > 0 && len(pageIDs) > 0 {
+		pageID := pageIDs[0]
+		pageIDs = pageIDs[1:]
+
+		if len(pageIDs) > 0 {
+			header.NextPage = pageIDs[0]
+		} else {
+			header.NextPage = 0
+		}
+
+		n := page.Write(data)
+		data = data[n:]
+
+		page.Header().CRC = page.ComputeCRC()
+
+		if err := pf.writePage(page, pageID); err != nil {
+			return err
+		}
+
+		// All but first page has pageTypeData.
+		header.PageType = pageTypeData
+	}
+
+	if len(pageIDs) > 0 {
+		return errs.Unexpected.WithMsg("Too many pages provided for given data.")
+	}
+
+	if len(data) > 0 {
+		return errs.Unexpected.WithMsg("Not enough pages for given data.")
+	}
+
+	return nil
+}
+
+func (pf *File) writePage(page dataPage, id uint64) error {
+	if _, err := pf.f.WriteAt(page, int64(id*pageSize)); err != nil {
+		return errs.IO.WithErr(err)
+	}
+	return nil
+}
+
+// ----------------------------------------------------------------------------
+// Reading
+// ----------------------------------------------------------------------------
+
+func (pf *File) iterate(each func(pageID uint64, page dataPage) error) error {
+	pf.lock.RLock()
+	defer pf.lock.RUnlock()
+
+	page := pf.page
+
+	fi, err := pf.f.Stat()
+	if err != nil {
+		return errs.IO.WithErr(err)
+	}
+
+	fileSize := fi.Size()
+	if fileSize%pageSize != 0 {
+		return errs.Corrupt.WithMsg("File size isn't a multiple of page size.")
+	}
+
+	maxPage := uint64(fileSize / pageSize)
+	if _, err := pf.f.Seek(0, io.SeekStart); err != nil {
+		return errs.IO.WithErr(err)
+	}
+
+	r := bufio.NewReaderSize(pf.f, 1024*1024)
+
+	for pageID := uint64(0); pageID < maxPage; pageID++ {
+		if _, err := r.Read(page); err != nil {
+			return errs.IO.WithErr(err)
+		}
+
+		if err := page.Validate(); err != nil {
+			return err
+		}
+		if err := each(pageID, page); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (pf *File) readData(id uint64, buf *bytes.Buffer) error {
+	page := pf.page
+
+	// The head page.
+	if err := pf.readPage(page, id); err != nil {
+		return err
+	}
+
+	remaining := int(page.Header().DataSize)
+
+	for {
+		data := page.Data()
+		if len(data) > remaining {
+			data = data[:remaining]
+		}
+
+		buf.Write(data)
+		remaining -= len(data)
+
+		if page.Header().NextPage == 0 {
+			break
+		}
+
+		if err := pf.readPage(page, page.Header().NextPage); err != nil {
+			return err
+		}
+	}
+
+	if remaining != 0 {
+		return errs.Corrupt.WithMsg("Incorrect data size. %d remaining.", remaining)
+	}
+
+	return nil
+}
+
+func (pf *File) readPage(p dataPage, id uint64) error {
+	if _, err := pf.f.ReadAt(p, int64(id*pageSize)); err != nil {
+		return errs.IO.WithErr(err)
+	}
+	return p.Validate()
+}
+
+// ----------------------------------------------------------------------------
+// Send / Recv
+// ----------------------------------------------------------------------------
+
+func (pf *File) Send(conn net.Conn, timeout time.Duration) error {
+	pf.lock.RLock()
+	defer pf.lock.RUnlock()
+
+	if _, err := pf.f.Seek(0, io.SeekStart); err != nil {
+		return errs.IO.WithErr(err)
+	}
+
+	fi, err := pf.f.Stat()
+	if err != nil {
+		return errs.IO.WithErr(err)
+	}
+
+	remaining := fi.Size()
+
+	conn.SetWriteDeadline(time.Now().Add(timeout))
+	if err := binary.Write(conn, binary.LittleEndian, remaining); err != nil {
+		return err
+	}
+
+	buf := make([]byte, 1024*1024)
+	w, err := gzip.NewWriterLevel(conn, 3)
+	if err != nil {
+		return errs.Unexpected.WithErr(err)
+	}
+	defer w.Close()
+
+	for remaining > 0 {
+		n, err := pf.f.Read(buf)
+		if err != nil {
+			return errs.IO.WithErr(err)
+		}
+
+		conn.SetWriteDeadline(time.Now().Add(timeout))
+		if _, err := w.Write(buf[:n]); err != nil {
+			return errs.IO.WithErr(err)
+		}
+
+		remaining -= int64(n)
+		w.Flush()
+	}
+
+	return nil
+}
+
+func Recv(conn net.Conn, filePath string, timeout time.Duration) error {
+	defer conn.Close()
+
+	f, err := os.Create(filePath)
+	if err != nil {
+		return errs.IO.WithErr(err)
+	}
+	defer f.Close()
+
+	remaining := uint64(0)
+	if err := binary.Read(conn, binary.LittleEndian, &remaining); err != nil {
+		return err
+	}
+
+	r, err := gzip.NewReader(conn)
+	if err != nil {
+		return errs.Unexpected.WithErr(err)
+	}
+	defer r.Close()
+
+	buf := make([]byte, 1024*1024)
+	for remaining > 0 {
+
+		conn.SetReadDeadline(time.Now().Add(timeout))
+
+		n, err := io.ReadFull(r, buf)
+		if err != nil && n == 0 {
+			return errs.IO.WithErr(err)
+		}
+		remaining -= uint64(n)
+
+		if _, err := f.Write(buf[:n]); err != nil {
+			return errs.IO.WithErr(err)
+		}
+	}
+
+	if err := f.Sync(); err != nil {
+		return errs.IO.WithErr(err)
+	}
+
+	return nil
+}

mdb/pfile/pagefile_test.go

@@ -0,0 +1,94 @@
+package pfile
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+type FileState struct {
+	SeqNum uint64
+	Data   map[[2]uint64][]byte
+}
+
+func (pf *File) Assert(t *testing.T, state pFileState) {
+	t.Helper()
+
+	pf.lock.RLock()
+	defer pf.lock.RUnlock()
+
+	idx, err := NewIndex(pf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	data := map[[2]uint64][]byte{}
+	err = IterateAllocated(pf, idx, func(cID, iID uint64, fileData []byte) error {
+		data[[2]uint64{cID, iID}] = bytes.Clone(fileData)
+		return nil
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(data) != len(state.Data) {
+		t.Fatalf("Expected %d items but got %d.", len(state.Data), len(data))
+	}
+
+	for key, expected := range state.Data {
+		val, ok := data[key]
+		if !ok {
+			t.Fatalf("No data found for key %v.", key)
+		}
+		if !bytes.Equal(val, expected) {
+			t.Fatalf("Incorrect data for key %v.", key)
+		}
+	}
+}
+
+func TestFileStateUpdateRandom(t *testing.T) {
+	t.Parallel()
+	tmpDir := t.TempDir()
+	walDir := filepath.Join(tmpDir, "wal")
+	pageFilePath := filepath.Join(tmpDir, "pagefile")
+
+	if err := os.MkdirAll(walDir, 0700); err != nil {
+		t.Fatal(err)
+	}
+
+	pf, err := Open(pageFilePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	idx, err := NewIndex(pf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	state := pFileState{
+		Data: map[[2]uint64][]byte{},
+	}
+
+	for i := uint64(1); i < 256; i++ {
+		changes := randomChangeList()
+		idx.StageChanges(changes)
+
+		if err := pf.ApplyChanges(changes); err != nil {
+			t.Fatal(err)
+		}
+		idx.ApplyChanges(changes)
+
+		for _, ch := range changes {
+			if !ch.Store {
+				delete(state.Data, [2]uint64{ch.CollectionID, ch.ItemID})
+			} else {
+				state.Data[[2]uint64{ch.CollectionID, ch.ItemID}] = ch.Data
+			}
+		}
+
+		pf.Assert(t, state)
+	}
+}

mdb/pfile/record_test.go

@@ -0,0 +1,57 @@
+package pfile
+
+import (
+	"bytes"
+	"git.crumpington.com/public/jldb/lib/wal"
+	"git.crumpington.com/public/jldb/mdb/change"
+)
+
+// ----------------------------------------------------------------------------
+
+type pFileState struct {
+	Data map[[2]uint64][]byte
+}
+
+// ----------------------------------------------------------------------------
+
+type recBuilder struct {
+	changes []change.Change
+	rec     wal.Record
+}
+
+func NewRecBuilder(seqNum, timestamp int64) *recBuilder {
+	return &recBuilder{
+		rec: wal.Record{
+			SeqNum:      seqNum,
+			TimestampMS: timestamp,
+		},
+		changes: []change.Change{},
+	}
+}
+
+func (b *recBuilder) Store(cID, iID uint64, data string) *recBuilder {
+	b.changes = append(b.changes, change.Change{
+		CollectionID: cID,
+		ItemID:       iID,
+		Store:        true,
+		Data:         []byte(data),
+	})
+	return b
+}
+
+func (b *recBuilder) Delete(cID, iID uint64) *recBuilder {
+	b.changes = append(b.changes, change.Change{
+		CollectionID: cID,
+		ItemID:       iID,
+		Store:        false,
+	})
+	return b
+}
+
+func (b *recBuilder) Record() wal.Record {
+	buf := &bytes.Buffer{}
+	change.Write(b.changes, buf)
+	b.rec.DataSize = int64(buf.Len())
+	b.rec.Reader = buf
+	return b.rec
+}

mdb/pfile/sendrecv_test.go

@@ -0,0 +1,62 @@
+package pfile
+
+/*
+func TestSendRecv(t *testing.T) {
+	tmpDir := t.TempDir()
+	filePath1 := filepath.Join(tmpDir, "1")
+	filePath2 := filepath.Join(tmpDir, "2")
+	defer os.RemoveAll(tmpDir)
+
+	f1, err := os.Create(filePath1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	size := rand.Int63n(1024 * 1024 * 128)
+	buf := make([]byte, size)
+	crand.Read(buf)
+	if _, err := f1.Write(buf); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := f1.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	c1, c2 := net.Pipe()
+	errChan := make(chan error)
+
+	go func() {
+		err := Send(filePath1, c1, time.Second)
+		if err != nil {
+			log.Printf("Send error: %v", err)
+		}
+		errChan <- err
+	}()
+
+	go func() {
+		err := Recv(filePath2, c2, time.Second)
+		if err != nil {
+			log.Printf("Recv error: %v", err)
+		}
+		errChan <- err
+	}()
+
+	if err := <-errChan; err != nil {
+		t.Fatal(err)
+	}
+
+	if err := <-errChan; err != nil {
+		t.Fatal(err)
+	}
+
+	buf2, err := os.ReadFile(filePath2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(buf, buf2) {
+		t.Fatal("Not equal.")
+	}
+}
+*/