README.md

@@ -4,7 +4,6 @@ Replicated in-memory database and file store.
 
 ## TODO
 
-* [ ] mdb: Tests for using `nil` snapshots ?
 * [ ] mdb: tests for sanitize and validate functions
 * [ ] Test: lib/wal iterator w/ corrupt file (random corruptions)
 * [ ] Test: lib/wal io.go

go.mod

@@ -1,10 +1,9 @@
 module git.crumpington.com/public/jldb
 
-go 1.22
+go 1.21.1
 
 require (
 	github.com/google/btree v1.1.2
-	go.uber.org/goleak v1.3.0
 	golang.org/x/net v0.15.0
 	golang.org/x/sys v0.12.0
 )

go.sum

@@ -1,16 +1,6 @@
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
 github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
-go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

lib/errs/error.go

@@ -8,27 +8,26 @@ import (
 )
 
 type Error struct {
-	Code       int64
+	msg        string
-	Msg        string
+	code       int64
-	StackTrace string
-
 	collection string
 	index      string
+	stackTrace string
 	err        error // Wrapped error
 }
 
 func NewErr(code int64, msg string) *Error {
 	return &Error{
-		Code: code,
+		msg:  msg,
-		Msg:  msg,
+		code: code,
 	}
 }
 
 func (e *Error) Error() string {
 	if e.collection != "" || e.index != "" {
-		return fmt.Sprintf(`[%d] (%s/%s) %s`, e.Code, e.collection, e.index, e.Msg)
+		return fmt.Sprintf(`[%d] (%s/%s) %s`, e.code, e.collection, e.index, e.msg)
 	} else {
-		return fmt.Sprintf("[%d] %s", e.Code, e.Msg)
+		return fmt.Sprintf("[%d] %s", e.code, e.msg)
 	}
 }
 
@@ -37,15 +36,11 @@ func (e *Error) Is(rhs error) bool {
 	if !ok {
 		return false
 	}
-	return e.Code == e2.Code
+	return e.code == e2.code
-}
-
-func (e *Error) Unwrap() error {
-	return e.err
 }
 
 func (e *Error) WithErr(err error) *Error {
-	if e2, ok := err.(*Error); ok && e2.Code == e.Code {
+	if e2, ok := err.(*Error); ok && e2.code == e.code {
 		return e2
 	}
 
@@ -54,11 +49,18 @@ func (e *Error) WithErr(err error) *Error {
 	return e2
 }
 
+func (e *Error) Unwrap() error {
+	if e.err != nil {
+		return e.err
+	}
+	return e
+}
+
 func (e *Error) WithMsg(msg string, args ...any) *Error {
 	err := *e
-	err.Msg += ": " + fmt.Sprintf(msg, args...)
+	err.msg += ": " + fmt.Sprintf(msg, args...)
-	if len(err.StackTrace) == 0 {
+	if len(err.stackTrace) == 0 {
-		err.StackTrace = string(debug.Stack())
+		err.stackTrace = string(debug.Stack())
 	}
 	return &err
 }
@@ -76,16 +78,16 @@ func (e *Error) WithIndex(s string) *Error {
 }
 
 func (e *Error) msgTruncacted() string {
-	if len(e.Msg) > 255 {
+	if len(e.msg) > 255 {
-		return e.Msg[:255]
+		return e.msg[:255]
 	}
-	return e.Msg
+	return e.msg
 }
 
 func (e *Error) Write(w io.Writer) error {
 	msg := e.msgTruncacted()
 
-	if err := binary.Write(w, binary.LittleEndian, e.Code); err != nil {
+	if err := binary.Write(w, binary.LittleEndian, e.code); err != nil {
 		return IO.WithErr(err)
 	}
 
@@ -101,7 +103,7 @@ func (e *Error) Read(r io.Reader) error {
 		size uint8
 	)
 
-	if err := binary.Read(r, binary.LittleEndian, &e.Code); err != nil {
+	if err := binary.Read(r, binary.LittleEndian, &e.code); err != nil {
 		return IO.WithErr(err)
 	}
 
@@ -114,6 +116,6 @@ func (e *Error) Read(r io.Reader) error {
 		return IO.WithErr(err)
 	}
 
-	e.Msg = string(msgBuf)
+	e.msg = string(msgBuf)
 	return nil
 }

lib/errs/fmt.go

@@ -10,12 +10,12 @@ func FmtDetails(err error) string {
 
 	var s string
 	if e.collection != "" || e.index != "" {
-		s = fmt.Sprintf(`[%d] (%s/%s) %s`, e.Code, e.collection, e.index, e.Msg)
+		s = fmt.Sprintf(`[%d] (%s/%s) %s`, e.code, e.collection, e.index, e.msg)
 	} else {
-		s = fmt.Sprintf("[%d] %s", e.Code, e.Msg)
+		s = fmt.Sprintf("[%d] %s", e.code, e.msg)
 	}
-	if len(e.StackTrace) != 0 {
+	if len(e.stackTrace) != 0 {
-		s += "\n\nStack Trace:\n" + e.StackTrace + "\n"
+		s += "\n\nStack Trace:\n" + e.stackTrace + "\n"
 	}
 
 	return s

lib/rep/main_test.go

@@ -1,11 +0,0 @@
-package rep
-
-import (
-	"testing"
-
-	"go.uber.org/goleak"
-)
-
-func TestMain(m *testing.M) {
-	goleak.VerifyTestMain(m)
-}

lib/rep/replicator-walgc.go

@@ -15,7 +15,7 @@ func (rep *Replicator) runWALGC() {
 		select {
 		case <-ticker.C:
 			state := rep.getState()
-			before := time.Now().Unix() - rep.conf.WALSegGCAgeSec
+			before := time.Now().Unix() - rep.conf.WALSegMaxAgeSec
 			if err := rep.wal.DeleteBefore(before, state.SeqNum); err != nil {
 				log.Printf("[WAL-GC] failed to delete wal segments: %v", err)
 			}

lib/rep/replicator.go

@@ -2,7 +2,6 @@ package rep
 
 import (
 	"io"
-	"log"
 	"net"
 	"os"
 	"sync"
@@ -36,8 +35,8 @@ type App struct {
 	// SendState: The primary may need to send storage state to a secondary node.
 	SendState func(conn net.Conn) error
 
-	// (1) RecvState: Secondary nodes may need to load state from the primary if
+	// (1) RecvState: Secondary nodes may need to load state from the primary if the
-	// the WAL is too far behind.
+	// WAL is too far behind.
 	RecvState func(conn net.Conn) error
 
 	// (2) InitStorage: Prepare application storage for possible calls to
@@ -95,49 +94,41 @@ func Open(app App, conf Config) (*Replicator, error) {
 	rep.client = newClient(rep.conf.PrimaryEndpoint, rep.conf.ReplicationPSK, rep.conf.NetTimeout)
 
 	if err := rep.initDirectories(); err != nil {
-		log.Printf("Failed to init directories: %v", err)
 		return nil, err
 	}
 
 	if err := rep.acquireLock(); err != nil {
 		rep.Close()
-		log.Printf("Failed to acquire lock: %v", err)
 		return nil, err
 	}
 
 	if err := rep.loadLocalState(); err != nil {
 		rep.Close()
-		log.Printf("Failed to load local state: %v", err)
 		return nil, err
 	}
 
 	if err := rep.openWAL(); err != nil {
 		rep.Close()
-		log.Printf("Failed to open WAL: %v", err)
 		return nil, err
 	}
 
 	if err := rep.recvStateIfNecessary(); err != nil {
 		rep.Close()
-		log.Printf("Failed to recv state: %v", err)
 		return nil, err
 	}
 
 	if err := rep.app.InitStorage(); err != nil {
 		rep.Close()
-		log.Printf("Failed to init storage: %v", err)
 		return nil, err
 	}
 
 	if err := rep.replay(); err != nil {
 		rep.Close()
-		log.Printf("Failed to replay: %v", err)
 		return nil, err
 	}
 
 	if err := rep.app.LoadFromStorage(); err != nil {
 		rep.Close()
-		log.Printf("Failed to load from storage: %v", err)
 		return nil, err
 	}
 

lib/rep/testapp-harness_test.go

@@ -56,7 +56,6 @@ func (h TestAppHarness) Run(t *testing.T) {
 				WALSegMaxAgeSec: 1,
 				WALSegGCAgeSec:  1,
 			})
-			defer app2.Close()
 
 			val.MethodByName(method.Name).Call([]reflect.Value{
 				reflect.ValueOf(t),

lib/wal/main_test.go

@@ -1,11 +0,0 @@
-package wal
-
-import (
-	"testing"
-
-	"go.uber.org/goleak"
-)
-
-func TestMain(m *testing.M) {
-	goleak.VerifyTestMain(m)
-}

mdb/collection.go

@@ -1,6 +1,7 @@
 package mdb
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"hash/crc64"
@@ -24,6 +25,8 @@ type Collection[T any] struct {
 	uniqueIndices []*Index[T]
 
 	ByID *Index[T]
+
+	buf *bytes.Buffer
 }
 
 type CollectionConfig[T any] struct {
@@ -64,6 +67,7 @@ func NewCollection[T any](db *Database, name string, conf *CollectionConfig[T])
 		validate:      conf.Validate,
 		indices:       []*Index[T]{},
 		uniqueIndices: []*Index[T]{},
+		buf:           &bytes.Buffer{},
 	}
 
 	db.addCollection(c.collectionID, c, &collectionState[T]{
@@ -104,6 +108,33 @@ type indexConfig[T any] struct {
 	Include func(item *T) bool
 }
 
+// A shortcut for c.ByID.Get(tx, &Item{ID:id}).
+func (c *Collection[T]) Get(tx *Snapshot, id uint64) (*T, bool) {
+	x := new(T)
+	c.setID(x, id)
+	return c.ByID.Get(tx, x)
+}
+
+func (c *Collection[T]) GetList(tx *Snapshot, ids []uint64, out []*T) []*T {
+	if len(ids) == 0 {
+		return out[:0]
+	}
+
+	if cap(out) < len(ids) {
+		out = make([]*T, len(ids))
+	}
+	out = out[:0]
+
+	for _, id := range ids {
+		item, ok := c.Get(tx, id)
+		if ok {
+			out = append(out, item)
+		}
+	}
+
+	return out
+}
+
 // AddIndex: Add an index to the collection.
 func (c *Collection[T]) addIndex(conf indexConfig[T]) *Index[T] {
 	var less func(*T, *T) bool
@@ -130,7 +161,6 @@ func (c *Collection[T]) addIndex(conf indexConfig[T]) *Index[T] {
 	}
 
 	index := &Index[T]{
-		db:           c.db,
 		collectionID: c.collectionID,
 		name:         conf.Name,
 		indexID:      c.getState(c.db.Snapshot()).addIndex(indexState),
@@ -146,35 +176,7 @@ func (c *Collection[T]) addIndex(conf indexConfig[T]) *Index[T] {
 	return index
 }
 
-func (c *Collection[T]) Get(tx *Snapshot, id uint64) *T {
-	if tx == nil {
-		tx = c.db.Snapshot()
-	}
-	item := new(T)
-	c.setID(item, id)
-	return c.ByID.Get(tx, item)
-}
-
-func (c *Collection[T]) Has(tx *Snapshot, id uint64) bool {
-	if tx == nil {
-		tx = c.db.Snapshot()
-	}
-	item := new(T)
-	c.setID(item, id)
-	return c.ByID.Has(tx, item)
-}
-
 func (c *Collection[T]) Insert(tx *Snapshot, userItem *T) error {
-	if tx == nil {
-		return c.db.Update(func(tx *Snapshot) error {
-			return c.insert(tx, userItem)
-		})
-	}
-
-	return c.insert(tx, userItem)
-}
-
-func (c *Collection[T]) insert(tx *Snapshot, userItem *T) error {
 	if err := c.ensureMutable(tx); err != nil {
 		return err
 	}
@@ -202,15 +204,6 @@ func (c *Collection[T]) insert(tx *Snapshot, userItem *T) error {
 }
 
 func (c *Collection[T]) Update(tx *Snapshot, userItem *T) error {
-	if tx == nil {
-		return c.db.Update(func(tx *Snapshot) error {
-			return c.update(tx, userItem)
-		})
-	}
-	return c.update(tx, userItem)
-}
-
-func (c *Collection[T]) update(tx *Snapshot, userItem *T) error {
 	if err := c.ensureMutable(tx); err != nil {
 		return err
 	}
@@ -243,36 +236,18 @@ func (c *Collection[T]) update(tx *Snapshot, userItem *T) error {
 }
 
 func (c *Collection[T]) UpdateFunc(tx *Snapshot, id uint64, update func(item *T) error) error {
-	if tx == nil {
+	item, ok := c.getByID(tx, id)
-		return c.db.Update(func(tx *Snapshot) error {
+	if !ok {
-			return c.updateFunc(tx, id, update)
-		})
-	}
-	return c.updateFunc(tx, id, update)
-}
-
-func (c *Collection[T]) updateFunc(tx *Snapshot, id uint64, update func(item *T) error) error {
-	item := c.Get(tx, id)
-	if item == nil {
 		return errs.NotFound
 	}
-	if err := update(item); err != nil {
+	itemCopy := c.copy(item)
+	if err := update(itemCopy); err != nil {
 		return err
 	}
-	c.setID(item, id) // Don't allow the ID to change.
+	return c.Update(tx, itemCopy)
-	return c.update(tx, item)
 }
 
 func (c *Collection[T]) Upsert(tx *Snapshot, item *T) error {
-	if tx == nil {
-		return c.db.Update(func(tx *Snapshot) error {
-			return c.upsert(tx, item)
-		})
-	}
-	return c.upsert(tx, item)
-}
-
-func (c *Collection[T]) upsert(tx *Snapshot, item *T) error {
 	err := c.Insert(tx, item)
 	if err == nil {
 		return nil
@@ -284,45 +259,20 @@ func (c *Collection[T]) upsert(tx *Snapshot, item *T) error {
 }
 
 func (c *Collection[T]) UpsertFunc(tx *Snapshot, id uint64, update func(item *T) error) error {
-	if tx == nil {
+	item, ok := c.getByID(tx, id)
-		c.db.Update(func(tx *Snapshot) error {
+	if !ok {
-			return c.upsertFunc(tx, id, update)
-		})
-	}
-	return c.upsertFunc(tx, id, update)
-}
-
-func (c *Collection[T]) upsertFunc(tx *Snapshot, id uint64, update func(item *T) error) error {
-	insert := false
-
-	item := c.Get(tx, id)
-	if item == nil {
 		item = new(T)
-		insert = true
+		c.setID(item, id)
 	}
 
-	if err := update(item); err != nil {
+	itemCopy := c.copy(item)
+	if err := update(itemCopy); err != nil {
 		return err
 	}
-
+	return c.Upsert(tx, itemCopy)
-	c.setID(item, id) // Don't allow the ID to change.
-
-	if insert {
-		return c.insert(tx, item)
-	}
-	return c.update(tx, item)
 }
 
 func (c *Collection[T]) Delete(tx *Snapshot, itemID uint64) error {
-	if tx == nil {
-		return c.db.Update(func(tx *Snapshot) error {
-			return c.delete(tx, itemID)
-		})
-	}
-	return c.delete(tx, itemID)
-}
-
-func (c *Collection[T]) delete(tx *Snapshot, itemID uint64) error {
 	if err := c.ensureMutable(tx); err != nil {
 		return err
 	}
@@ -330,13 +280,6 @@ func (c *Collection[T]) delete(tx *Snapshot, itemID uint64) error {
 	return c.deleteItem(tx, itemID)
 }
 
-func (c *Collection[T]) Count(tx *Snapshot) int {
-	if tx == nil {
-		tx = c.db.Snapshot()
-	}
-	return c.ByID.Count(tx)
-}
-
 func (c *Collection[T]) getByID(tx *Snapshot, itemID uint64) (*T, bool) {
 	x := new(T)
 	c.setID(x, itemID)

mdb/db-primary.go

@@ -1,67 +0,0 @@
-package mdb
-
-/*
-func (db *Database) openPrimary() (err error) {
-	wal, err := cwal.Open(db.walRootDir, cwal.Config{
-		SegMinCount:  db.conf.WALSegMinCount,
-		SegMaxAgeSec: db.conf.WALSegMaxAgeSec,
-	})
-
-	pFile, err := pfile.Open(db.pageFilePath,
-
-	pFile, err := openPageFileAndReplayWAL(db.rootDir)
-	if err != nil {
-		return err
-	}
-	defer pFile.Close()
-
-	pfHeader, err := pFile.ReadHeader()
-	if err != nil {
-		return err
-	}
-
-	tx := db.Snapshot()
-	tx.seqNum = pfHeader.SeqNum
-	tx.updatedAt = pfHeader.UpdatedAt
-
-	pIndex, err := pagefile.NewIndex(pFile)
-	if err != nil {
-		return err
-	}
-
-	err = pFile.IterateAllocated(pIndex, func(cID, iID uint64, data []byte) error {
-		return db.loadItem(tx, cID, iID, data)
-	})
-	if err != nil {
-		return err
-	}
-
-	w, err := cwal.OpenWriter(db.walRootDir, &cwal.WriterConfig{
-		SegMinCount:  db.conf.WALSegMinCount,
-		SegMaxAgeSec: db.conf.WALSegMaxAgeSec,
-	})
-	if err != nil {
-		return err
-	}
-
-	db.done.Add(1)
-	go txAggregator{
-		Stop:     db.stop,
-		Done:     db.done,
-		ModChan:  db.modChan,
-		W:        w,
-		Index:    pIndex,
-		Snapshot: db.snapshot,
-	}.Run()
-
-	db.done.Add(1)
-	go (&fileWriter{
-		Stop:         db.stop,
-		Done:         db.done,
-		PageFilePath: db.pageFilePath,
-		WALRootDir:   db.walRootDir,
-	}).Run()
-
-	return nil
-}
-*/

mdb/db-rep.go

@@ -99,7 +99,6 @@ func (db *Database) repApply(rec wal.Record) (err error) {
 	}
 	tx.seqNum = rec.SeqNum
 	tx.timestampMS = rec.TimestampMS
-	tx.setReadOnly()
 	db.snapshot.Store(tx)
 	return nil
 }

mdb/db-secondary.go

@@ -1,129 +0,0 @@
-package mdb
-
-/*
-func (db *Database) openSecondary() (err error) {
-	if db.shouldLoadFromPrimary() {
-		if err := db.loadFromPrimary(); err != nil {
-			return err
-		}
-	}
-
-	log.Printf("Opening page-file...")
-
-	pFile, err := openPageFileAndReplayWAL(db.rootDir)
-	if err != nil {
-		return err
-	}
-	defer pFile.Close()
-
-	pfHeader, err := pFile.ReadHeader()
-	if err != nil {
-		return err
-	}
-
-	log.Printf("Building page-file index...")
-
-	pIndex, err := pagefile.NewIndex(pFile)
-	if err != nil {
-		return err
-	}
-
-	tx := db.Snapshot()
-	tx.seqNum = pfHeader.SeqNum
-	tx.updatedAt = pfHeader.UpdatedAt
-
-	log.Printf("Loading data into memory...")
-
-	err = pFile.IterateAllocated(pIndex, func(cID, iID uint64, data []byte) error {
-		return db.loadItem(tx, cID, iID, data)
-	})
-	if err != nil {
-		return err
-	}
-
-	log.Printf("Creating writer...")
-
-	w, err := cswal.OpenWriter(db.walRootDir, &cswal.WriterConfig{
-		SegMinCount:  db.conf.WALSegMinCount,
-		SegMaxAgeSec: db.conf.WALSegMaxAgeSec,
-	})
-	if err != nil {
-		return err
-	}
-
-	db.done.Add(1)
-	go (&walFollower{
-		Stop:   db.stop,
-		Done:   db.done,
-		W:      w,
-		Client: NewClient(db.conf.PrimaryURL, db.conf.ReplicationPSK, db.conf.NetTimeout),
-	}).Run()
-
-	db.done.Add(1)
-	go (&follower{
-		Stop:         db.stop,
-		Done:         db.done,
-		WALRootDir:   db.walRootDir,
-		SeqNum:       pfHeader.SeqNum,
-		ApplyChanges: db.applyChanges,
-	}).Run()
-
-	db.done.Add(1)
-	go (&fileWriter{
-		Stop:         db.stop,
-		Done:         db.done,
-		PageFilePath: db.pageFilePath,
-		WALRootDir:   db.walRootDir,
-	}).Run()
-
-	return nil
-}
-
-func (db *Database) shouldLoadFromPrimary() bool {
-	if _, err := os.Stat(db.walRootDir); os.IsNotExist(err) {
-		log.Printf("WAL doesn't exist.")
-		return true
-	}
-	if _, err := os.Stat(db.pageFilePath); os.IsNotExist(err) {
-		log.Printf("Page-file doesn't exist.")
-		return true
-	}
-	return false
-}
-
-func (db *Database) loadFromPrimary() error {
-	client := NewClient(db.conf.PrimaryURL, db.conf.ReplicationPSK, db.conf.NetTimeout)
-	defer client.Disconnect()
-
-	log.Printf("Loading data from primary...")
-
-	if err := os.RemoveAll(db.pageFilePath); err != nil {
-		log.Printf("Failed to remove page-file: %s", err)
-		return errs.IO.WithErr(err) // Caller can retry.
-	}
-
-	if err := os.RemoveAll(db.walRootDir); err != nil {
-		log.Printf("Failed to remove WAL: %s", err)
-		return errs.IO.WithErr(err) // Caller can retry.
-	}
-
-	err := client.DownloadPageFile(db.pageFilePath+".tmp", db.pageFilePath)
-	if err != nil {
-		log.Printf("Failed to get page-file from primary: %s", err)
-		return err // Caller can retry.
-	}
-
-	pfHeader, err := pagefile.ReadHeader(db.pageFilePath)
-	if err != nil {
-		log.Printf("Failed to read page-file sequence number: %s", err)
-		return err // Caller can retry.
-	}
-
-	if err = cswal.CreateEx(db.walRootDir, pfHeader.SeqNum+1); err != nil {
-		log.Printf("Failed to initialize WAL: %s", err)
-		return err // Caller can retry.
-	}
-
-	return nil
-}
-*/

mdb/db-testcases_test.go

@@ -54,8 +54,8 @@ var testDBTestCases = []DBTestCase{{
 		Name: "Update",
 
 		Update: func(t *testing.T, db TestDB, tx *Snapshot) error {
-			user := db.Users.ByID.Get(tx, &User{ID: 1})
+			user, ok := db.Users.ByID.Get(tx, &User{ID: 1})
-			if user == nil {
+			if !ok {
 				return errs.NotFound
 			}
 			user.Name = "Bob"
@@ -323,8 +323,8 @@ var testDBTestCases = []DBTestCase{{
 		Name: "Update",
 
 		Update: func(t *testing.T, db TestDB, tx *Snapshot) error {
-			user := db.Users.ByID.Get(tx, &User{ID: 1})
+			user, ok := db.Users.ByID.Get(tx, &User{ID: 1})
-			if user == nil {
+			if !ok {
 				return errs.NotFound
 			}
 			user.Name = "Bob"
@@ -493,8 +493,8 @@ var testDBTestCases = []DBTestCase{{
 		Name: "Update",
 
 		Update: func(t *testing.T, db TestDB, tx *Snapshot) error {
-			u := db.Users.ByID.Get(tx, &User{ID: 2})
+			u, ok := db.Users.ByID.Get(tx, &User{ID: 2})
-			if u == nil {
+			if !ok {
 				return errs.NotFound
 			}
 
@@ -609,16 +609,16 @@ var testDBTestCases = []DBTestCase{{
 		Update: func(t *testing.T, db TestDB, tx *Snapshot) error {
 			expected := &User{ID: 1, Name: "Alice", Email: "a@b.com"}
 
-			u := db.Users.ByID.Get(tx, &User{ID: 1})
+			u, ok := db.Users.ByID.Get(tx, &User{ID: 1})
-			if u == nil {
+			if !ok {
 				return errs.NotFound
 			}
 			if !reflect.DeepEqual(u, expected) {
 				return errors.New("Not equal (id)")
 			}
 
-			u = db.Users.ByEmail.Get(tx, &User{Email: "a@b.com"})
+			u, ok = db.Users.ByEmail.Get(tx, &User{Email: "a@b.com"})
-			if u == nil {
+			if !ok {
 				return errs.NotFound
 			}
 			if !reflect.DeepEqual(u, expected) {
@@ -637,11 +637,11 @@ var testDBTestCases = []DBTestCase{{
 		Name: "Get not found",
 
 		Update: func(t *testing.T, db TestDB, tx *Snapshot) error {
-			if u := db.Users.ByID.Get(tx, &User{ID: 2}); u != nil {
+			if _, ok := db.Users.ByID.Get(tx, &User{ID: 2}); ok {
 				return errors.New("Found (id)")
 			}
 
-			if u := db.Users.ByEmail.Get(tx, &User{Email: "x@b.com"}); u != nil {
+			if _, ok := db.Users.ByEmail.Get(tx, &User{Email: "x@b.com"}); ok {
 				return errors.New("Found (email)")
 			}
 
@@ -743,25 +743,29 @@ var testDBTestCases = []DBTestCase{{
 
 			first := true
 			pivot := User{Name: "User1"}
-			for u := range db.Users.ByName.AscendAfter(tx, &pivot) {
+			db.Users.ByName.AscendAfter(tx, &pivot, func(u *User) bool {
 				u.Name += "Mod"
 				if err = db.Users.Update(tx, u); err != nil {
-					return err
+					return false
 				}
 				if first {
 					first = false
-					continue
+					return true
 				}
 
-				prev := db.Users.ByID.Get(tx, &User{ID: u.ID - 1})
+				prev, ok := db.Users.ByID.Get(tx, &User{ID: u.ID - 1})
-				if prev == nil {
+				if !ok {
-					return errors.New("Previous user not found")
+					err = errors.New("Previous user not found")
+					return false
 				}
 
 				if !strings.HasSuffix(prev.Name, "Mod") {
-					return errors.New("Incorrect user name: " + prev.Name)
+					err = errors.New("Incorrect user name: " + prev.Name)
-				}
+					return false
 				}
+
+				return true
+			})
 			return nil
 		},
 
@@ -797,26 +801,29 @@ var testDBTestCases = []DBTestCase{{
 			}
 
 			first := true
-			for u := range db.Users.ByName.DescendAfter(tx, &User{Name: "User5Mod"}) {
+			db.Users.ByName.DescendAfter(tx, &User{Name: "User5Mod"}, func(u *User) bool {
 				u.Name = strings.TrimSuffix(u.Name, "Mod")
 				if err = db.Users.Update(tx, u); err != nil {
-					return err
+					return false
 				}
 				if first {
 					first = false
-					continue
+					return true
 				}
 
-				prev := db.Users.ByID.Get(tx, &User{ID: u.ID + 1})
+				prev, ok := db.Users.ByID.Get(tx, &User{ID: u.ID + 1})
-				if prev == nil {
+				if !ok {
-					return errors.New("Previous user not found")
+					err = errors.New("Previous user not found")
+					return false
 				}
 
 				if strings.HasSuffix(prev.Name, "Mod") {
-					return errors.New("Incorrect user name: " + prev.Name)
+					err = errors.New("Incorrect user name: " + prev.Name)
-				}
+					return false
 				}
 
+				return true
+			})
 			return nil
 		},
 

mdb/db-testrunner_test.go

@@ -72,7 +72,7 @@ func testRunner_testCase(t *testing.T, testCase DBTestCase) {
 	}
 
 	// TODO: Why is this necessary?
-	//time.Sleep(time.Second)
+	time.Sleep(time.Second)
 	finalStep := testCase.Steps[len(testCase.Steps)-1]
 
 	secondarySnapshot := db2.Snapshot()
@@ -136,21 +136,21 @@ func checkSlicesEqual[T any](t *testing.T, name string, actual, expected []T) {
 
 func checkMinMaxEqual[T any](t *testing.T, name string, tx *Snapshot, index *Index[T], expected []T) {
 	if len(expected) == 0 {
-		if min := index.Min(tx); min != nil {
+		if min, ok := index.Min(tx); ok {
 			t.Fatal(min)
 		}
-		if max := index.Max(tx); max != nil {
+		if max, ok := index.Max(tx); ok {
 			t.Fatal(max)
 		}
 		return
 	}
 
-	min := index.Min(tx)
+	min, ok := index.Min(tx)
-	if min == nil {
+	if !ok {
 		t.Fatal("No min")
 	}
-	max := index.Max(tx)
+	max, ok := index.Max(tx)
-	if max == nil {
+	if !ok {
 		t.Fatal("No max")
 	}
 

mdb/db.go

@@ -73,10 +73,6 @@ type Database struct {
 }
 
 func New(conf Config) *Database {
-	if conf.NetTimeout <= 0 {
-		conf.NetTimeout = time.Minute
-	}
-
 	if conf.MaxConcurrentUpdates <= 0 {
 		conf.MaxConcurrentUpdates = 32
 	}

mdb/equality_test.go

@@ -1,6 +1,7 @@
 package mdb
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
 )
@@ -19,16 +20,18 @@ func (i Index[T]) AssertEqual(t *testing.T, tx1, tx2 *Snapshot) {
 	}
 
 	errStr := ""
-	iter := i.Ascend(tx1)
+	i.Ascend(tx1, func(item1 *T) bool {
-	for item1 := range iter {
+		item2, ok := i.Get(tx2, item1)
-		item2 := i.Get(tx2, item1)
+		if !ok {
-		if item2 == nil {
+			errStr = fmt.Sprintf("Indices don't match. %v not found.", item1)
-			t.Fatalf("Indices don't match. %v not found.", item1)
+			return false
 		}
 		if !reflect.DeepEqual(item1, item2) {
-			t.Fatalf("%v != %v", item1, item2)
+			errStr = fmt.Sprintf("%v != %v", item1, item2)
-		}
+			return false
 		}
+		return true
+	})
 
 	if errStr != "" {
 		t.Fatal(errStr)

mdb/index.go

@@ -1,7 +1,6 @@
 package mdb
 
 import (
-	"iter"
 	"unsafe"
 
 	"github.com/google/btree"
@@ -64,7 +63,6 @@ func NewUniquePartialIndex[T any](
 // ----------------------------------------------------------------------------
 
 type Index[T any] struct {
-	db           *Database
 	name         string
 	collectionID uint64
 	indexID      uint64
@@ -72,19 +70,12 @@ type Index[T any] struct {
 	copy         func(*T) *T
 }
 
-func (i *Index[T]) ensureSnapshot(tx *Snapshot) *Snapshot {
+func (i *Index[T]) Get(tx *Snapshot, in *T) (item *T, ok bool) {
-	if tx == nil {
+	tPtr, ok := i.get(tx, in)
-		tx = i.db.Snapshot()
+	if !ok {
+		return item, false
 	}
-	return tx
+	return i.copy(tPtr), true
-}
-
-func (i *Index[T]) Get(tx *Snapshot, in *T) *T {
-	tx = i.ensureSnapshot(tx)
-	if tPtr, ok := i.get(tx, in); ok {
-		return i.copy(tPtr)
-	}
-	return nil
 }
 
 func (i *Index[T]) get(tx *Snapshot, in *T) (*T, bool) {
@@ -92,65 +83,95 @@ func (i *Index[T]) get(tx *Snapshot, in *T) (*T, bool) {
 }
 
 func (i *Index[T]) Has(tx *Snapshot, in *T) bool {
-	tx = i.ensureSnapshot(tx)
 	return i.btree(tx).Has(in)
 }
 
-func (i *Index[T]) Min(tx *Snapshot) *T {
+func (i *Index[T]) Min(tx *Snapshot) (item *T, ok bool) {
-	tx = i.ensureSnapshot(tx)
+	tPtr, ok := i.btree(tx).Min()
-	if tPtr, ok := i.btree(tx).Min(); ok {
+	if !ok {
-		return i.copy(tPtr)
+		return item, false
 	}
-	return nil
+	return i.copy(tPtr), true
 }
 
-func (i *Index[T]) Max(tx *Snapshot) *T {
+func (i *Index[T]) Max(tx *Snapshot) (item *T, ok bool) {
-	tx = i.ensureSnapshot(tx)
+	tPtr, ok := i.btree(tx).Max()
-	if tPtr, ok := i.btree(tx).Max(); ok {
+	if !ok {
-		return i.copy(tPtr)
+		return item, false
 	}
-	return nil
+	return i.copy(tPtr), true
 }
 
-func (i *Index[T]) Ascend(tx *Snapshot) iter.Seq[*T] {
+func (i *Index[T]) Ascend(tx *Snapshot, each func(*T) bool) {
-	tx = i.ensureSnapshot(tx)
-	return func(yield func(*T) bool) {
 	i.btreeForIter(tx).Ascend(func(t *T) bool {
-			return yield(i.copy(t))
+		return each(i.copy(t))
 	})
 }
-}
 
-func (i *Index[T]) AscendAfter(tx *Snapshot, after *T) iter.Seq[*T] {
+func (i *Index[T]) AscendAfter(tx *Snapshot, after *T, each func(*T) bool) {
-	tx = i.ensureSnapshot(tx)
-	return func(yield func(*T) bool) {
 	i.btreeForIter(tx).AscendGreaterOrEqual(after, func(t *T) bool {
-			return yield(i.copy(t))
+		return each(i.copy(t))
 	})
 }
-}
 
-func (i *Index[T]) Descend(tx *Snapshot) iter.Seq[*T] {
+func (i *Index[T]) Descend(tx *Snapshot, each func(*T) bool) {
-	tx = i.ensureSnapshot(tx)
-	return func(yield func(*T) bool) {
 	i.btreeForIter(tx).Descend(func(t *T) bool {
-			return yield(i.copy(t))
+		return each(i.copy(t))
 	})
 }
-}
 
-func (i *Index[T]) DescendAfter(tx *Snapshot, after *T) iter.Seq[*T] {
+func (i *Index[T]) DescendAfter(tx *Snapshot, after *T, each func(*T) bool) {
-	tx = i.ensureSnapshot(tx)
-	return func(yield func(*T) bool) {
 	i.btreeForIter(tx).DescendLessOrEqual(after, func(t *T) bool {
-			return yield(i.copy(t))
+		return each(i.copy(t))
 	})
 }
+
+type ListArgs[T any] struct {
+	Desc  bool          // True for descending order, otherwise ascending.
+	After *T            // If after is given, iterate after (and including) the value.
+	While func(*T) bool // Continue iterating until While is false.
+	Limit int           // Maximum number of items to return. 0 => All.
+}
+
+func (i *Index[T]) List(tx *Snapshot, args ListArgs[T], out []*T) []*T {
+	if args.Limit < 0 {
+		return nil
+	}
+
+	if args.While == nil {
+		args.While = func(*T) bool { return true }
+	}
+
+	size := args.Limit
+	if size == 0 {
+		size = 32 // Why not?
+	}
+
+	items := out[:0]
+
+	each := func(item *T) bool {
+		if !args.While(item) {
+			return false
+		}
+		items = append(items, item)
+		return args.Limit == 0 || len(items) < args.Limit
+	}
+
+	if args.Desc {
+		if args.After != nil {
+			i.DescendAfter(tx, args.After, each)
+		} else {
+			i.Descend(tx, each)
+		}
+	} else {
+		if args.After != nil {
+			i.AscendAfter(tx, args.After, each)
+		} else {
+			i.Ascend(tx, each)
+		}
 	}
 
-func (i *Index[T]) Count(tx *Snapshot) int {
+	return items
-	tx = i.ensureSnapshot(tx)
-	return i.btree(tx).Len()
 }
 
 // ----------------------------------------------------------------------------

mdb/index_test.go

@@ -1,8 +1,9 @@
 package mdb
 
 func (i Index[T]) Dump(tx *Snapshot) (l []T) {
-	for t := range i.Ascend(tx) {
+	i.Ascend(tx, func(t *T) bool {
 		l = append(l, *t)
-	}
+		return true
+	})
 	return l
 }

mdb/main_test.go

@@ -1,11 +0,0 @@
-package mdb
-
-import (
-	"testing"
-
-	"go.uber.org/goleak"
-)
-
-func TestMain(m *testing.M) {
-	goleak.VerifyTestMain(m)
-}

mdb/pfile/freelist.go

@@ -51,10 +51,6 @@ func (f *freeList) Push(pages ...uint64) {
 	}
 }
 
-func (f *freeList) SetNextPage(nextPage uint64) {
-	f.nextPage = nextPage
-}
-
 func (f *freeList) Pop(count int, out []uint64) []uint64 {
 	out = out[:0]
 

mdb/pfile/index.go

@@ -13,19 +13,14 @@ type Index struct {
 }
 
 func NewIndex(f *File) (*Index, error) {
-	firstPage, err := f.pageCount()
-	if err != nil {
-		return nil, err
-	}
-
 	idx := &Index{
-		fList: newFreeList(firstPage),
+		fList: newFreeList(0),
 		aList: *newAllocList(),
 		seen:  map[[2]uint64]struct{}{},
 		mask:  []bool{},
 	}
 
-	err = f.iterate(func(pageID uint64, page dataPage) error {
+	err := f.iterate(func(pageID uint64, page dataPage) error {
 		header := page.Header()
 		switch header.PageType {
 		case pageTypeHead:

mdb/pfile/pagefile.go

@@ -134,21 +134,6 @@ func (pf *File) writePage(page dataPage, id uint64) error {
 // Reading
 // ----------------------------------------------------------------------------
 
-func (pf *File) pageCount() (uint64, error) {
-	fi, err := pf.f.Stat()
-	if err != nil {
-		return 0, errs.IO.WithErr(err)
-	}
-
-	fileSize := fi.Size()
-	if fileSize%pageSize != 0 {
-		return 0, errs.Corrupt.WithMsg("File size isn't a multiple of page size.")
-	}
-
-	maxPage := uint64(fileSize / pageSize)
-	return maxPage, nil
-}
-
 func (pf *File) iterate(each func(pageID uint64, page dataPage) error) error {
 	pf.lock.RLock()
 	defer pf.lock.RUnlock()

mdb/testdb_test.go

@@ -133,8 +133,8 @@ func (db DataDB) modifyOnce() {
 func (db DataDB) ComputeCRC(tx *Snapshot) uint32 {
 	h := crc32.NewIEEE()
 	for dataID := uint64(1); dataID < 10; dataID++ {
-		d := db.Datas.ByID.Get(tx, &DataItem{ID: dataID})
+		d, ok := db.Datas.ByID.Get(tx, &DataItem{ID: dataID})
-		if d == nil {
+		if !ok {
 			continue
 		}
 		h.Write(d.Data)
@@ -143,8 +143,8 @@ func (db DataDB) ComputeCRC(tx *Snapshot) uint32 {
 }
 
 func (db DataDB) ReadCRC(tx *Snapshot) uint32 {
-	r := db.CRCs.ByID.Get(tx, &CRCItem{ID: 1})
+	r, ok := db.CRCs.ByID.Get(tx, &CRCItem{ID: 1})
-	if r == nil {
+	if !ok {
 		return 0
 	}
 	return r.CRC32

mdb/testing/crashconsistency/main.go

@@ -136,8 +136,8 @@ func (db DataDB) modifyOnce() {
 func (db DataDB) ComputeCRC(tx *mdb.Snapshot) uint32 {
 	h := crc32.NewIEEE()
 	for dataID := uint64(1); dataID < 10; dataID++ {
-		d := db.Datas.ByID.Get(tx, &DataItem{ID: dataID})
+		d, ok := db.Datas.ByID.Get(tx, &DataItem{ID: dataID})
-		if d == nil {
+		if !ok {
 			continue
 		}
 		h.Write(d.Data)
@@ -146,8 +146,8 @@ func (db DataDB) ComputeCRC(tx *mdb.Snapshot) uint32 {
 }
 
 func (db DataDB) ReadCRC(tx *mdb.Snapshot) uint32 {
-	r := db.CRCs.ByID.Get(tx, &CRCItem{ID: 1})
+	r, ok := db.CRCs.ByID.Get(tx, &CRCItem{ID: 1})
-	if r == nil {
+	if !ok {
 		return 0
 	}
 	return r.CRC32