public/jldb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 17 Wiki Activity

Compare commits

base: public:master
Branches Tags
public:master
public:v0.10.0
public:v0.9.0
public:v0.8.1
public:v0.8.0
public:v0.7.3
public:v0.7.2
public:v0.7.1
public:v0.7.0
public:v0.6.3
public:v0.6.2
public:v0.6.1
public:v0.6.0
public:v0.5.0
public:v0.4.0
public:v0.3.0
public:v0.2.0
public:v0.1.0
..
compare: public:v0.1.0
Branches Tags
public:master
public:v0.10.0
public:v0.9.0
public:v0.8.1
public:v0.8.0
public:v0.7.3
public:v0.7.2
public:v0.7.1
public:v0.7.0
public:v0.6.3
public:v0.6.2
public:v0.6.1
public:v0.6.0
public:v0.5.0
public:v0.4.0
public:v0.3.0
public:v0.2.0
public:v0.1.0

No commits in common. "master" and "v0.1.0" have entirely different histories.
master ... v0.1.0

26 changed files with 246 additions and 379 deletions
Show Stats Expand all files Collapse all files

1
README.md
View File

@ -4,7 +4,6 @@ Replicated in-memory database and file store.
## TODO
* [ ] mdb: Tests for using `nil` snapshots ?
* [ ] mdb: tests for sanitize and validate functions
* [ ] Test: lib/wal iterator w/ corrupt file (random corruptions)
* [ ] Test: lib/wal io.go

3
go.mod
View File

@ -1,10 +1,9 @@
module git.crumpington.com/public/jldb
go 1.22
go 1.21.1
require (
github.com/google/btree v1.1.2
go.uber.org/goleak v1.3.0
golang.org/x/net v0.15.0
golang.org/x/sys v0.12.0
)

10
go.sum
View File

@ -1,16 +1,6 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

48
lib/errs/error.go
View File

@ -8,27 +8,26 @@ import (
)
type Error struct {
Code int64
Msg string
StackTrace string
msg string
code int64
collection string
index string
stackTrace string
err error // Wrapped error
}
func NewErr(code int64, msg string) *Error {
return &Error{
Code: code,
Msg: msg,
msg: msg,
code: code,
}
}
func (e *Error) Error() string {
if e.collection != "" || e.index != "" {
return fmt.Sprintf(`[%d] (%s/%s) %s`, e.Code, e.collection, e.index, e.Msg)
return fmt.Sprintf(`[%d] (%s/%s) %s`, e.code, e.collection, e.index, e.msg)
} else {
return fmt.Sprintf("[%d] %s", e.Code, e.Msg)
return fmt.Sprintf("[%d] %s", e.code, e.msg)
}
}
@ -37,15 +36,11 @@ func (e *Error) Is(rhs error) bool {
if !ok {
return false
}
return e.Code == e2.Code
}
func (e *Error) Unwrap() error {
return e.err
return e.code == e2.code
}
func (e *Error) WithErr(err error) *Error {
if e2, ok := err.(*Error); ok && e2.Code == e.Code {
if e2, ok := err.(*Error); ok && e2.code == e.code {
return e2
}
@ -54,11 +49,18 @@ func (e *Error) WithErr(err error) *Error {
return e2
}
func (e *Error) Unwrap() error {
if e.err != nil {
return e.err
}
return e
}
func (e *Error) WithMsg(msg string, args ...any) *Error {
err := *e
err.Msg += ": " + fmt.Sprintf(msg, args...)
if len(err.StackTrace) == 0 {
err.StackTrace = string(debug.Stack())
err.msg += ": " + fmt.Sprintf(msg, args...)
if len(err.stackTrace) == 0 {
err.stackTrace = string(debug.Stack())
}
return &err
}
@ -76,16 +78,16 @@ func (e *Error) WithIndex(s string) *Error {
}
func (e *Error) msgTruncacted() string {
if len(e.Msg) > 255 {
return e.Msg[:255]
if len(e.msg) > 255 {
return e.msg[:255]
}
return e.Msg
return e.msg
}
func (e *Error) Write(w io.Writer) error {
msg := e.msgTruncacted()
if err := binary.Write(w, binary.LittleEndian, e.Code); err != nil {
if err := binary.Write(w, binary.LittleEndian, e.code); err != nil {
return IO.WithErr(err)
}
@ -101,7 +103,7 @@ func (e *Error) Read(r io.Reader) error {
size uint8
)
if err := binary.Read(r, binary.LittleEndian, &e.Code); err != nil {
if err := binary.Read(r, binary.LittleEndian, &e.code); err != nil {
return IO.WithErr(err)
}
@ -114,6 +116,6 @@ func (e *Error) Read(r io.Reader) error {
return IO.WithErr(err)
}
e.Msg = string(msgBuf)
e.msg = string(msgBuf)
return nil
}

8
lib/errs/fmt.go
View File

@ -10,12 +10,12 @@ func FmtDetails(err error) string {
var s string
if e.collection != "" || e.index != "" {
s = fmt.Sprintf(`[%d] (%s/%s) %s`, e.Code, e.collection, e.index, e.Msg)
s = fmt.Sprintf(`[%d] (%s/%s) %s`, e.code, e.collection, e.index, e.msg)
} else {
s = fmt.Sprintf("[%d] %s", e.Code, e.Msg)
s = fmt.Sprintf("[%d] %s", e.code, e.msg)
}
if len(e.StackTrace) != 0 {
s += "\n\nStack Trace:\n" + e.StackTrace + "\n"
if len(e.stackTrace) != 0 {
s += "\n\nStack Trace:\n" + e.stackTrace + "\n"
}
return s

11
lib/rep/main_test.go
View File

@ -1,11 +0,0 @@
package rep
import (
"testing"
"go.uber.org/goleak"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}

2
lib/rep/replicator-walgc.go
View File

@ -15,7 +15,7 @@ func (rep *Replicator) runWALGC() {
select {
case <-ticker.C:
state := rep.getState()
before := time.Now().Unix() - rep.conf.WALSegGCAgeSec
before := time.Now().Unix() - rep.conf.WALSegMaxAgeSec
if err := rep.wal.DeleteBefore(before, state.SeqNum); err != nil {
log.Printf("[WAL-GC] failed to delete wal segments: %v", err)
}

13
lib/rep/replicator.go
View File

@ -2,7 +2,6 @@ package rep
import (
"io"
"log"
"net"
"os"
"sync"
@ -36,8 +35,8 @@ type App struct {
// SendState: The primary may need to send storage state to a secondary node.
SendState func(conn net.Conn) error
// (1) RecvState: Secondary nodes may need to load state from the primary if
// the WAL is too far behind.
// (1) RecvState: Secondary nodes may need to load state from the primary if the
// WAL is too far behind.
RecvState func(conn net.Conn) error
// (2) InitStorage: Prepare application storage for possible calls to
@ -95,49 +94,41 @@ func Open(app App, conf Config) (*Replicator, error) {
rep.client = newClient(rep.conf.PrimaryEndpoint, rep.conf.ReplicationPSK, rep.conf.NetTimeout)
if err := rep.initDirectories(); err != nil {
log.Printf("Failed to init directories: %v", err)
return nil, err
}
if err := rep.acquireLock(); err != nil {
rep.Close()
log.Printf("Failed to acquire lock: %v", err)
return nil, err
}
if err := rep.loadLocalState(); err != nil {
rep.Close()
log.Printf("Failed to load local state: %v", err)
return nil, err
}
if err := rep.openWAL(); err != nil {
rep.Close()
log.Printf("Failed to open WAL: %v", err)
return nil, err
}
if err := rep.recvStateIfNecessary(); err != nil {
rep.Close()
log.Printf("Failed to recv state: %v", err)
return nil, err
}
if err := rep.app.InitStorage(); err != nil {
rep.Close()
log.Printf("Failed to init storage: %v", err)
return nil, err
}
if err := rep.replay(); err != nil {
rep.Close()
log.Printf("Failed to replay: %v", err)
return nil, err
}
if err := rep.app.LoadFromStorage(); err != nil {
rep.Close()
log.Printf("Failed to load from storage: %v", err)
return nil, err
}

1
lib/rep/testapp-harness_test.go
View File

@ -56,7 +56,6 @@ func (h TestAppHarness) Run(t *testing.T) {
WALSegMaxAgeSec: 1,
WALSegGCAgeSec: 1,
})
defer app2.Close()
val.MethodByName(method.Name).Call([]reflect.Value{
reflect.ValueOf(t),

11
lib/wal/main_test.go
View File

@ -1,11 +0,0 @@
package wal
import (
"testing"
"go.uber.org/goleak"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}

185
mdb/collection.go
View File

@ -1,6 +1,7 @@
package mdb
import (
"bytes"
"encoding/json"
"errors"
"hash/crc64"
@ -20,10 +21,12 @@ type Collection[T any] struct {
sanitize func(*T)
validate func(*T) error
indices []*Index[T]
uniqueIndices []*Index[T]
indices []Index[T]
uniqueIndices []Index[T]
ByID *Index[T]
ByID Index[T]
buf *bytes.Buffer
}
type CollectionConfig[T any] struct {
@ -62,8 +65,9 @@ func NewCollection[T any](db *Database, name string, conf *CollectionConfig[T])
copy: conf.Copy,
sanitize: conf.Sanitize,
validate: conf.Validate,
indices: []*Index[T]{},
uniqueIndices: []*Index[T]{},
indices: []Index[T]{},
uniqueIndices: []Index[T]{},
buf: &bytes.Buffer{},
}
db.addCollection(c.collectionID, c, &collectionState[T]{
@ -88,7 +92,7 @@ func NewCollection[T any](db *Database, name string, conf *CollectionConfig[T])
return c
}
func (c *Collection[T]) Name() string {
func (c Collection[T]) Name() string {
return c.name
}
@ -104,8 +108,35 @@ type indexConfig[T any] struct {
Include func(item *T) bool
}
func (c Collection[T]) Get(tx *Snapshot, id uint64) (*T, bool) {
x := new(T)
c.setID(x, id)
return c.ByID.Get(tx, x)
}
func (c Collection[T]) List(tx *Snapshot, ids []uint64, out []*T) []*T {
if len(ids) == 0 {
return out[:0]
}
if cap(out) < len(ids) {
out = make([]*T, len(ids))
}
out = out[:0]
for _, id := range ids {
item, ok := c.Get(tx, id)
if ok {
out = append(out, item)
}
}
return out
}
// AddIndex: Add an index to the collection.
func (c *Collection[T]) addIndex(conf indexConfig[T]) *Index[T] {
func (c *Collection[T]) addIndex(conf indexConfig[T]) Index[T] {
var less func(*T, *T) bool
if conf.Unique {
@ -129,8 +160,7 @@ func (c *Collection[T]) addIndex(conf indexConfig[T]) *Index[T] {
BTree: btree.NewG(256, less),
}
index := &Index[T]{
db: c.db,
index := Index[T]{
collectionID: c.collectionID,
name: conf.Name,
indexID: c.getState(c.db.Snapshot()).addIndex(indexState),
@ -146,35 +176,7 @@ func (c *Collection[T]) addIndex(conf indexConfig[T]) *Index[T] {
return index
}
func (c *Collection[T]) Get(tx *Snapshot, id uint64) *T {
if tx == nil {
tx = c.db.Snapshot()
}
item := new(T)
c.setID(item, id)
return c.ByID.Get(tx, item)
}
func (c *Collection[T]) Has(tx *Snapshot, id uint64) bool {
if tx == nil {
tx = c.db.Snapshot()
}
item := new(T)
c.setID(item, id)
return c.ByID.Has(tx, item)
}
func (c *Collection[T]) Insert(tx *Snapshot, userItem *T) error {
if tx == nil {
return c.db.Update(func(tx *Snapshot) error {
return c.insert(tx, userItem)
})
}
return c.insert(tx, userItem)
}
func (c *Collection[T]) insert(tx *Snapshot, userItem *T) error {
func (c Collection[T]) Insert(tx *Snapshot, userItem *T) error {
if err := c.ensureMutable(tx); err != nil {
return err
}
@ -201,16 +203,7 @@ func (c *Collection[T]) insert(tx *Snapshot, userItem *T) error {
return nil
}
func (c *Collection[T]) Update(tx *Snapshot, userItem *T) error {
if tx == nil {
return c.db.Update(func(tx *Snapshot) error {
return c.update(tx, userItem)
})
}
return c.update(tx, userItem)
}
func (c *Collection[T]) update(tx *Snapshot, userItem *T) error {
func (c Collection[T]) Update(tx *Snapshot, userItem *T) error {
if err := c.ensureMutable(tx); err != nil {
return err
}
@ -242,37 +235,7 @@ func (c *Collection[T]) update(tx *Snapshot, userItem *T) error {
return nil
}
func (c *Collection[T]) UpdateFunc(tx *Snapshot, id uint64, update func(item *T) error) error {
if tx == nil {
return c.db.Update(func(tx *Snapshot) error {
return c.updateFunc(tx, id, update)
})
}
return c.updateFunc(tx, id, update)
}
func (c *Collection[T]) updateFunc(tx *Snapshot, id uint64, update func(item *T) error) error {
item := c.Get(tx, id)
if item == nil {
return errs.NotFound
}
if err := update(item); err != nil {
return err
}
c.setID(item, id) // Don't allow the ID to change.
return c.update(tx, item)
}
func (c *Collection[T]) Upsert(tx *Snapshot, item *T) error {
if tx == nil {
return c.db.Update(func(tx *Snapshot) error {
return c.upsert(tx, item)
})
}
return c.upsert(tx, item)
}
func (c *Collection[T]) upsert(tx *Snapshot, item *T) error {
func (c Collection[T]) Upsert(tx *Snapshot, item *T) error {
err := c.Insert(tx, item)
if err == nil {
return nil
@ -283,46 +246,7 @@ func (c *Collection[T]) upsert(tx *Snapshot, item *T) error {
return err
}
func (c *Collection[T]) UpsertFunc(tx *Snapshot, id uint64, update func(item *T) error) error {
if tx == nil {
c.db.Update(func(tx *Snapshot) error {
return c.upsertFunc(tx, id, update)
})
}
return c.upsertFunc(tx, id, update)
}
func (c *Collection[T]) upsertFunc(tx *Snapshot, id uint64, update func(item *T) error) error {
insert := false
item := c.Get(tx, id)
if item == nil {
item = new(T)
insert = true
}
if err := update(item); err != nil {
return err
}
c.setID(item, id) // Don't allow the ID to change.
if insert {
return c.insert(tx, item)
}
return c.update(tx, item)
}
func (c *Collection[T]) Delete(tx *Snapshot, itemID uint64) error {
if tx == nil {
return c.db.Update(func(tx *Snapshot) error {
return c.delete(tx, itemID)
})
}
return c.delete(tx, itemID)
}
func (c *Collection[T]) delete(tx *Snapshot, itemID uint64) error {
func (c Collection[T]) Delete(tx *Snapshot, itemID uint64) error {
if err := c.ensureMutable(tx); err != nil {
return err
}
@ -330,20 +254,13 @@ func (c *Collection[T]) delete(tx *Snapshot, itemID uint64) error {
return c.deleteItem(tx, itemID)
}
func (c *Collection[T]) Count(tx *Snapshot) int {
if tx == nil {
tx = c.db.Snapshot()
}
return c.ByID.Count(tx)
}
func (c *Collection[T]) getByID(tx *Snapshot, itemID uint64) (*T, bool) {
func (c Collection[T]) getByID(tx *Snapshot, itemID uint64) (*T, bool) {
x := new(T)
c.setID(x, itemID)
return c.ByID.get(tx, x)
}
func (c *Collection[T]) ensureMutable(tx *Snapshot) error {
func (c Collection[T]) ensureMutable(tx *Snapshot) error {
if !tx.writable() {
return errs.ReadOnly
}
@ -357,7 +274,7 @@ func (c *Collection[T]) ensureMutable(tx *Snapshot) error {
}
// For initial data loading.
func (c *Collection[T]) insertItem(tx *Snapshot, itemID uint64, data []byte) error {
func (c Collection[T]) insertItem(tx *Snapshot, itemID uint64, data []byte) error {
item := new(T)
if err := json.Unmarshal(data, item); err != nil {
return errs.Encoding.WithErr(err).WithCollection(c.name)
@ -378,7 +295,7 @@ func (c *Collection[T]) insertItem(tx *Snapshot, itemID uint64, data []byte) err
return nil
}
func (c *Collection[T]) deleteItem(tx *Snapshot, itemID uint64) error {
func (c Collection[T]) deleteItem(tx *Snapshot, itemID uint64) error {
item, ok := c.getByID(tx, itemID)
if !ok {
return errs.NotFound
@ -395,7 +312,7 @@ func (c *Collection[T]) deleteItem(tx *Snapshot, itemID uint64) error {
// upsertItem inserts or updates the item with itemID and the given serialized
// form. It's called by
func (c *Collection[T]) upsertItem(tx *Snapshot, itemID uint64, data []byte) error {
func (c Collection[T]) upsertItem(tx *Snapshot, itemID uint64, data []byte) error {
item, ok := c.getByID(tx, itemID)
if ok {
tx.delete(c.collectionID, itemID)
@ -418,14 +335,14 @@ func (c *Collection[T]) upsertItem(tx *Snapshot, itemID uint64, data []byte) err
return nil
}
func (c *Collection[T]) getID(t *T) uint64 {
func (c Collection[T]) getID(t *T) uint64 {
return *((*uint64)(unsafe.Pointer(t)))
}
func (c *Collection[T]) setID(t *T, id uint64) {
func (c Collection[T]) setID(t *T, id uint64) {
*((*uint64)(unsafe.Pointer(t))) = id
}
func (c *Collection[T]) getState(tx *Snapshot) *collectionState[T] {
func (c Collection[T]) getState(tx *Snapshot) *collectionState[T] {
return tx.collections[c.collectionID].(*collectionState[T])
}

1
mdb/db-rep.go
View File

@ -99,7 +99,6 @@ func (db *Database) repApply(rec wal.Record) (err error) {
}
tx.seqNum = rec.SeqNum
tx.timestampMS = rec.TimestampMS
tx.setReadOnly()
db.snapshot.Store(tx)
return nil
}

63
mdb/db-testcases_test.go
View File

@ -54,8 +54,8 @@ var testDBTestCases = []DBTestCase{{
Name: "Update",
Update: func(t *testing.T, db TestDB, tx *Snapshot) error {
user := db.Users.ByID.Get(tx, &User{ID: 1})
if user == nil {
user, ok := db.Users.ByID.Get(tx, &User{ID: 1})
if !ok {
return errs.NotFound
}
user.Name = "Bob"
@ -323,8 +323,8 @@ var testDBTestCases = []DBTestCase{{
Name: "Update",
Update: func(t *testing.T, db TestDB, tx *Snapshot) error {
user := db.Users.ByID.Get(tx, &User{ID: 1})
if user == nil {
user, ok := db.Users.ByID.Get(tx, &User{ID: 1})
if !ok {
return errs.NotFound
}
user.Name = "Bob"
@ -493,8 +493,8 @@ var testDBTestCases = []DBTestCase{{
Name: "Update",
Update: func(t *testing.T, db TestDB, tx *Snapshot) error {
u := db.Users.ByID.Get(tx, &User{ID: 2})
if u == nil {
u, ok := db.Users.ByID.Get(tx, &User{ID: 2})
if !ok {
return errs.NotFound
}
@ -609,16 +609,16 @@ var testDBTestCases = []DBTestCase{{
Update: func(t *testing.T, db TestDB, tx *Snapshot) error {
expected := &User{ID: 1, Name: "Alice", Email: "a@b.com"}
u := db.Users.ByID.Get(tx, &User{ID: 1})
if u == nil {
u, ok := db.Users.ByID.Get(tx, &User{ID: 1})
if !ok {
return errs.NotFound
}
if !reflect.DeepEqual(u, expected) {
return errors.New("Not equal (id)")
}
u = db.Users.ByEmail.Get(tx, &User{Email: "a@b.com"})
if u == nil {
u, ok = db.Users.ByEmail.Get(tx, &User{Email: "a@b.com"})
if !ok {
return errs.NotFound
}
if !reflect.DeepEqual(u, expected) {
@ -637,11 +637,11 @@ var testDBTestCases = []DBTestCase{{
Name: "Get not found",
Update: func(t *testing.T, db TestDB, tx *Snapshot) error {
if u := db.Users.ByID.Get(tx, &User{ID: 2}); u != nil {
if _, ok := db.Users.ByID.Get(tx, &User{ID: 2}); ok {
return errors.New("Found (id)")
}
if u := db.Users.ByEmail.Get(tx, &User{Email: "x@b.com"}); u != nil {
if _, ok := db.Users.ByEmail.Get(tx, &User{Email: "x@b.com"}); ok {
return errors.New("Found (email)")
}
@ -743,25 +743,29 @@ var testDBTestCases = []DBTestCase{{
first := true
pivot := User{Name: "User1"}
for u := range db.Users.ByName.AscendAfter(tx, &pivot) {
db.Users.ByName.AscendAfter(tx, &pivot, func(u *User) bool {
u.Name += "Mod"
if err = db.Users.Update(tx, u); err != nil {
return err
return false
}
if first {
first = false
continue
return true
}
prev := db.Users.ByID.Get(tx, &User{ID: u.ID - 1})
if prev == nil {
return errors.New("Previous user not found")
prev, ok := db.Users.ByID.Get(tx, &User{ID: u.ID - 1})
if !ok {
err = errors.New("Previous user not found")
return false
}
if !strings.HasSuffix(prev.Name, "Mod") {
return errors.New("Incorrect user name: " + prev.Name)
err = errors.New("Incorrect user name: " + prev.Name)
return false
}
}
return true
})
return nil
},
@ -797,26 +801,29 @@ var testDBTestCases = []DBTestCase{{
}
first := true
for u := range db.Users.ByName.DescendAfter(tx, &User{Name: "User5Mod"}) {
db.Users.ByName.DescendAfter(tx, &User{Name: "User5Mod"}, func(u *User) bool {
u.Name = strings.TrimSuffix(u.Name, "Mod")
if err = db.Users.Update(tx, u); err != nil {
return err
return false
}
if first {
first = false
continue
return true
}
prev := db.Users.ByID.Get(tx, &User{ID: u.ID + 1})
if prev == nil {
return errors.New("Previous user not found")
prev, ok := db.Users.ByID.Get(tx, &User{ID: u.ID + 1})
if !ok {
err = errors.New("Previous user not found")
return false
}
if strings.HasSuffix(prev.Name, "Mod") {
return errors.New("Incorrect user name: " + prev.Name)
err = errors.New("Incorrect user name: " + prev.Name)
return false
}
}
return true
})
return nil
},

16
mdb/db-testrunner_test.go
View File

@ -72,7 +72,7 @@ func testRunner_testCase(t *testing.T, testCase DBTestCase) {
}
// TODO: Why is this necessary?
//time.Sleep(time.Second)
time.Sleep(time.Second)
finalStep := testCase.Steps[len(testCase.Steps)-1]
secondarySnapshot := db2.Snapshot()
@ -134,23 +134,23 @@ func checkSlicesEqual[T any](t *testing.T, name string, actual, expected []T) {
}
}
func checkMinMaxEqual[T any](t *testing.T, name string, tx *Snapshot, index *Index[T], expected []T) {
func checkMinMaxEqual[T any](t *testing.T, name string, tx *Snapshot, index Index[T], expected []T) {
if len(expected) == 0 {
if min := index.Min(tx); min != nil {
if min, ok := index.Min(tx); ok {
t.Fatal(min)
}
if max := index.Max(tx); max != nil {
if max, ok := index.Max(tx); ok {
t.Fatal(max)
}
return
}
min := index.Min(tx)
if min == nil {
min, ok := index.Min(tx)
if !ok {
t.Fatal("No min")
}
max := index.Max(tx)
if max == nil {
max, ok := index.Max(tx)
if !ok {
t.Fatal("No max")
}

2
mdb/db-userdata_test.go
View File

@ -14,7 +14,7 @@ type UserDataItem struct {
type UserData struct {
*Collection[UserDataItem]
ByName *Index[UserDataItem] // Unique index on (Token).
ByName Index[UserDataItem] // Unique index on (Token).
}
func NewUserDataCollection(db *Database) UserData {

6
mdb/db-users_test.go
View File

@ -12,9 +12,9 @@ type User struct {
type Users struct {
*Collection[User]
ByEmail *Index[User] // Unique index on (Email).
ByName *Index[User] // Index on (Name).
ByBlocked *Index[User] // Partial index on (Blocked,Email).
ByEmail Index[User] // Unique index on (Email).
ByName Index[User] // Index on (Name).
ByBlocked Index[User] // Partial index on (Blocked,Email).
}
func NewUserCollection(db *Database) Users {

4
mdb/db.go
View File

@ -73,10 +73,6 @@ type Database struct {
}
func New(conf Config) *Database {
if conf.NetTimeout <= 0 {
conf.NetTimeout = time.Minute
}
if conf.MaxConcurrentUpdates <= 0 {
conf.MaxConcurrentUpdates = 32
}

17
mdb/equality_test.go
View File

@ -1,6 +1,7 @@
package mdb
import (
"fmt"
"reflect"
"testing"
)
@ -19,16 +20,18 @@ func (i Index[T]) AssertEqual(t *testing.T, tx1, tx2 *Snapshot) {
}
errStr := ""
iter := i.Ascend(tx1)
for item1 := range iter {
item2 := i.Get(tx2, item1)
if item2 == nil {
t.Fatalf("Indices don't match. %v not found.", item1)
i.Ascend(tx1, func(item1 *T) bool {
item2, ok := i.Get(tx2, item1)
if !ok {
errStr = fmt.Sprintf("Indices don't match. %v not found.", item1)
return false
}
if !reflect.DeepEqual(item1, item2) {
t.Fatalf("%v != %v", item1, item2)
errStr = fmt.Sprintf("%v != %v", item1, item2)
return false
}
}
return true
})
if errStr != "" {
t.Fatal(errStr)

163
mdb/index.go
View File

@ -1,7 +1,6 @@
package mdb
import (
"iter"
"unsafe"
"github.com/google/btree"
@ -11,7 +10,7 @@ func NewIndex[T any](
c *Collection[T],
name string,
compare func(lhs, rhs *T) int,
) *Index[T] {
) Index[T] {
return c.addIndex(indexConfig[T]{
Name: name,
Unique: false,
@ -25,7 +24,7 @@ func NewPartialIndex[T any](
name string,
compare func(lhs, rhs *T) int,
include func(*T) bool,
) *Index[T] {
) Index[T] {
return c.addIndex(indexConfig[T]{
Name: name,
Unique: false,
@ -38,7 +37,7 @@ func NewUniqueIndex[T any](
c *Collection[T],
name string,
compare func(lhs, rhs *T) int,
) *Index[T] {
) Index[T] {
return c.addIndex(indexConfig[T]{
Name: name,
Unique: true,
@ -52,7 +51,7 @@ func NewUniquePartialIndex[T any](
name string,
compare func(lhs, rhs *T) int,
include func(*T) bool,
) *Index[T] {
) Index[T] {
return c.addIndex(indexConfig[T]{
Name: name,
Unique: true,
@ -64,7 +63,6 @@ func NewUniquePartialIndex[T any](
// ----------------------------------------------------------------------------
type Index[T any] struct {
db *Database
name string
collectionID uint64
indexID uint64
@ -72,94 +70,117 @@ type Index[T any] struct {
copy func(*T) *T
}
func (i *Index[T]) ensureSnapshot(tx *Snapshot) *Snapshot {
if tx == nil {
tx = i.db.Snapshot()
func (i Index[T]) Get(tx *Snapshot, in *T) (item *T, ok bool) {
tPtr, ok := i.get(tx, in)
if !ok {
return item, false
}
return tx
return i.copy(tPtr), true
}
func (i *Index[T]) Get(tx *Snapshot, in *T) *T {
tx = i.ensureSnapshot(tx)
if tPtr, ok := i.get(tx, in); ok {
return i.copy(tPtr)
}
return nil
}
func (i *Index[T]) get(tx *Snapshot, in *T) (*T, bool) {
func (i Index[T]) get(tx *Snapshot, in *T) (*T, bool) {
return i.btree(tx).Get(in)
}
func (i *Index[T]) Has(tx *Snapshot, in *T) bool {
tx = i.ensureSnapshot(tx)
func (i Index[T]) Has(tx *Snapshot, in *T) bool {
return i.btree(tx).Has(in)
}
func (i *Index[T]) Min(tx *Snapshot) *T {
tx = i.ensureSnapshot(tx)
if tPtr, ok := i.btree(tx).Min(); ok {
return i.copy(tPtr)
func (i Index[T]) Min(tx *Snapshot) (item *T, ok bool) {
tPtr, ok := i.btree(tx).Min()
if !ok {
return item, false
}
return nil
return i.copy(tPtr), true
}
func (i *Index[T]) Max(tx *Snapshot) *T {
tx = i.ensureSnapshot(tx)
if tPtr, ok := i.btree(tx).Max(); ok {
return i.copy(tPtr)
func (i Index[T]) Max(tx *Snapshot) (item *T, ok bool) {
tPtr, ok := i.btree(tx).Max()
if !ok {
return item, false
}
return nil
return i.copy(tPtr), true
}
func (i *Index[T]) Ascend(tx *Snapshot) iter.Seq[*T] {
tx = i.ensureSnapshot(tx)
return func(yield func(*T) bool) {
i.btreeForIter(tx).Ascend(func(t *T) bool {
return yield(i.copy(t))
})
}
func (i Index[T]) Ascend(tx *Snapshot, each func(*T) bool) {
i.btreeForIter(tx).Ascend(func(t *T) bool {
return each(i.copy(t))
})
}
func (i *Index[T]) AscendAfter(tx *Snapshot, after *T) iter.Seq[*T] {
tx = i.ensureSnapshot(tx)
return func(yield func(*T) bool) {
i.btreeForIter(tx).AscendGreaterOrEqual(after, func(t *T) bool {
return yield(i.copy(t))
})
}
func (i Index[T]) AscendAfter(tx *Snapshot, after *T, each func(*T) bool) {
i.btreeForIter(tx).AscendGreaterOrEqual(after, func(t *T) bool {
return each(i.copy(t))
})
}
func (i *Index[T]) Descend(tx *Snapshot) iter.Seq[*T] {
tx = i.ensureSnapshot(tx)
return func(yield func(*T) bool) {
i.btreeForIter(tx).Descend(func(t *T) bool {
return yield(i.copy(t))
})
}
func (i Index[T]) Descend(tx *Snapshot, each func(*T) bool) {
i.btreeForIter(tx).Descend(func(t *T) bool {
return each(i.copy(t))
})
}
func (i *Index[T]) DescendAfter(tx *Snapshot, after *T) iter.Seq[*T] {
tx = i.ensureSnapshot(tx)
return func(yield func(*T) bool) {
i.btreeForIter(tx).DescendLessOrEqual(after, func(t *T) bool {
return yield(i.copy(t))
})
}
func (i Index[T]) DescendAfter(tx *Snapshot, after *T, each func(*T) bool) {
i.btreeForIter(tx).DescendLessOrEqual(after, func(t *T) bool {
return each(i.copy(t))
})
}
func (i *Index[T]) Count(tx *Snapshot) int {
tx = i.ensureSnapshot(tx)
return i.btree(tx).Len()
type ListArgs[T any] struct {
Desc bool // True for descending order, otherwise ascending.
After *T // If after is given, iterate after (and including) the value.
While func(*T) bool // Continue iterating until While is false.
Limit int // Maximum number of items to return. 0 => All.
}
func (i Index[T]) List(tx *Snapshot, args ListArgs[T], out []*T) []*T {
if args.Limit < 0 {
return nil
}
if args.While == nil {
args.While = func(*T) bool { return true }
}
size := args.Limit
if size == 0 {
size = 32 // Why not?
}
items := out[:0]
each := func(item *T) bool {
if !args.While(item) {
return false
}
items = append(items, item)
return args.Limit == 0 || len(items) < args.Limit
}
if args.Desc {
if args.After != nil {
i.DescendAfter(tx, args.After, each)
} else {
i.Descend(tx, each)
}
} else {
if args.After != nil {
i.AscendAfter(tx, args.After, each)
} else {
i.Ascend(tx, each)
}
}
return items
}
// ----------------------------------------------------------------------------
func (i *Index[T]) insertConflict(tx *Snapshot, item *T) bool {
func (i Index[T]) insertConflict(tx *Snapshot, item *T) bool {
return i.btree(tx).Has(item)
}
func (i *Index[T]) updateConflict(tx *Snapshot, item *T) bool {
func (i Index[T]) updateConflict(tx *Snapshot, item *T) bool {
current, ok := i.btree(tx).Get(item)
return ok && i.getID(current) != i.getID(item)
}
@ -167,7 +188,7 @@ func (i *Index[T]) updateConflict(tx *Snapshot, item *T) bool {
// This should only be called after insertConflict. Additionally, the caller
// should ensure that the index has been properly cloned for write before
// writing.
func (i *Index[T]) insert(tx *Snapshot, item *T) {
func (i Index[T]) insert(tx *Snapshot, item *T) {
if i.include != nil && !i.include(item) {
return
}
@ -175,7 +196,7 @@ func (i *Index[T]) insert(tx *Snapshot, item *T) {
i.btree(tx).ReplaceOrInsert(item)
}
func (i *Index[T]) update(tx *Snapshot, old, new *T) {
func (i Index[T]) update(tx *Snapshot, old, new *T) {
bt := i.btree(tx)
bt.Delete(old)
@ -183,22 +204,22 @@ func (i *Index[T]) update(tx *Snapshot, old, new *T) {
i.insert(tx, new)
}
func (i *Index[T]) delete(tx *Snapshot, item *T) {
func (i Index[T]) delete(tx *Snapshot, item *T) {
i.btree(tx).Delete(item)
}
// ----------------------------------------------------------------------------
func (i *Index[T]) getState(tx *Snapshot) indexState[T] {
func (i Index[T]) getState(tx *Snapshot) indexState[T] {
return tx.collections[i.collectionID].(*collectionState[T]).Indices[i.indexID]
}
// Get the current btree for get/has/update/delete, etc.
func (i *Index[T]) btree(tx *Snapshot) *btree.BTreeG[*T] {
func (i Index[T]) btree(tx *Snapshot) *btree.BTreeG[*T] {
return i.getState(tx).BTree
}
func (i *Index[T]) btreeForIter(tx *Snapshot) *btree.BTreeG[*T] {
func (i Index[T]) btreeForIter(tx *Snapshot) *btree.BTreeG[*T] {
cState := tx.collections[i.collectionID].(*collectionState[T])
bt := cState.Indices[i.indexID].BTree
@ -210,6 +231,6 @@ func (i *Index[T]) btreeForIter(tx *Snapshot) *btree.BTreeG[*T] {
return bt
}
func (i *Index[T]) getID(t *T) uint64 {
func (i Index[T]) getID(t *T) uint64 {
return *((*uint64)(unsafe.Pointer(t)))
}

5
mdb/index_test.go
View File

@ -1,8 +1,9 @@
package mdb
func (i Index[T]) Dump(tx *Snapshot) (l []T) {
for t := range i.Ascend(tx) {
i.Ascend(tx, func(t *T) bool {
l = append(l, *t)
}
return true
})
return l
}

11
mdb/main_test.go
View File

@ -1,11 +0,0 @@
package mdb
import (
"testing"
"go.uber.org/goleak"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}

4
mdb/pfile/freelist.go
View File

@ -51,10 +51,6 @@ func (f *freeList) Push(pages ...uint64) {
}
}
func (f *freeList) SetNextPage(nextPage uint64) {
f.nextPage = nextPage
}
func (f *freeList) Pop(count int, out []uint64) []uint64 {
out = out[:0]

9
mdb/pfile/index.go
View File

@ -13,19 +13,14 @@ type Index struct {
}
func NewIndex(f *File) (*Index, error) {
firstPage, err := f.pageCount()
if err != nil {
return nil, err
}
idx := &Index{
fList: newFreeList(firstPage),
fList: newFreeList(0),
aList: *newAllocList(),
seen: map[[2]uint64]struct{}{},
mask: []bool{},
}
err = f.iterate(func(pageID uint64, page dataPage) error {
err := f.iterate(func(pageID uint64, page dataPage) error {
header := page.Header()
switch header.PageType {
case pageTypeHead:

15
mdb/pfile/pagefile.go
View File

@ -134,21 +134,6 @@ func (pf *File) writePage(page dataPage, id uint64) error {
// Reading
// ----------------------------------------------------------------------------
func (pf *File) pageCount() (uint64, error) {
fi, err := pf.f.Stat()
if err != nil {
return 0, errs.IO.WithErr(err)
}
fileSize := fi.Size()
if fileSize%pageSize != 0 {
return 0, errs.Corrupt.WithMsg("File size isn't a multiple of page size.")
}
maxPage := uint64(fileSize / pageSize)
return maxPage, nil
}
func (pf *File) iterate(each func(pageID uint64, page dataPage) error) error {
pf.lock.RLock()
defer pf.lock.RUnlock()

8
mdb/testdb_test.go
View File

@ -133,8 +133,8 @@ func (db DataDB) modifyOnce() {
func (db DataDB) ComputeCRC(tx *Snapshot) uint32 {
h := crc32.NewIEEE()
for dataID := uint64(1); dataID < 10; dataID++ {
d := db.Datas.ByID.Get(tx, &DataItem{ID: dataID})
if d == nil {
d, ok := db.Datas.ByID.Get(tx, &DataItem{ID: dataID})
if !ok {
continue
}
h.Write(d.Data)
@ -143,8 +143,8 @@ func (db DataDB) ComputeCRC(tx *Snapshot) uint32 {
}
func (db DataDB) ReadCRC(tx *Snapshot) uint32 {
r := db.CRCs.ByID.Get(tx, &CRCItem{ID: 1})
if r == nil {
r, ok := db.CRCs.ByID.Get(tx, &CRCItem{ID: 1})
if !ok {
return 0
}
return r.CRC32

8
mdb/testing/crashconsistency/main.go
View File

@ -136,8 +136,8 @@ func (db DataDB) modifyOnce() {
func (db DataDB) ComputeCRC(tx *mdb.Snapshot) uint32 {
h := crc32.NewIEEE()
for dataID := uint64(1); dataID < 10; dataID++ {
d := db.Datas.ByID.Get(tx, &DataItem{ID: dataID})
if d == nil {
d, ok := db.Datas.ByID.Get(tx, &DataItem{ID: dataID})
if !ok {
continue
}
h.Write(d.Data)
@ -146,8 +146,8 @@ func (db DataDB) ComputeCRC(tx *mdb.Snapshot) uint32 {
}
func (db DataDB) ReadCRC(tx *mdb.Snapshot) uint32 {
r := db.CRCs.ByID.Get(tx, &CRCItem{ID: 1})
if r == nil {
r, ok := db.CRCs.ByID.Get(tx, &CRCItem{ID: 1})
if !ok {
return 0
}
return r.CRC32