diff --git a/README.md b/README.md index 8b2217d..8acd02a 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ An in-process, in-memory database for Go. ## TO DO -* [ ] mdb: db exclusive lock -* [ ] mdb: clean up tests +* MDB Tests + * ## Structure diff --git a/btreeindex_ex_test.go b/btreeindex_ex_test.go new file mode 100644 index 0000000..0806d38 --- /dev/null +++ b/btreeindex_ex_test.go @@ -0,0 +1,51 @@ +package mdb + +import ( + "fmt" + "reflect" +) + +func (bt *BTreeIndex[T]) Equals(rhs *BTreeIndex[T]) error { + if bt.Len() != rhs.Len() { + return fmt.Errorf("Expected %d items, but found %d.", bt.Len(), rhs.Len()) + } + + it1 := bt.Ascend() + defer it1.Close() + + it2 := rhs.Ascend() + defer it2.Close() + + for it1.Next() { + it2.Next() + + v1 := it1.Value() + v2 := it2.Value() + + if !reflect.DeepEqual(v1, v2) { + return fmt.Errorf("Value mismatch: %v != %v", v1, v2) + } + } + + return nil +} + +func (bt *BTreeIndex[T]) EqualsList(data []*T) error { + if bt.Len() != len(data) { + return fmt.Errorf("Expected %d items, but found %d.", bt.Len(), len(data)) + } + + it1 := bt.Ascend() + defer it1.Close() + + for _, v1 := range data { + it1.Next() + v2 := it1.Value() + + if !reflect.DeepEqual(v1, v2) { + return fmt.Errorf("Value mismatch: %v != %v", v1, v2) + } + } + + return nil +} diff --git a/btreeindex_test.go b/btreeindex_test.go index 6d4adef..ac23bbf 100644 --- a/btreeindex_test.go +++ b/btreeindex_test.go @@ -1,13 +1,6 @@ package mdb -import ( - "fmt" - "os" - "path/filepath" - "reflect" - "testing" -) - +/* func TestBTreeIndex(t *testing.T) { type Item struct { ID uint64 @@ -380,3 +373,4 @@ func TestBTreeIndex(t *testing.T) { } }) } +*/ diff --git a/collection_test.go b/collection_test.go index 63c2751..4f5e826 100644 --- a/collection_test.go +++ b/collection_test.go @@ -1,17 +1,6 @@ package mdb -import ( - "errors" - "fmt" - "log" - "os" - "path/filepath" - "reflect" - "strings" - "sync" - "testing" -) - +/* func TestCollection(t *testing.T) { type Item struct { ID uint64 @@ -424,3 +413,4 @@ func BenchmarkLoad(b *testing.B) { panic("What?") } } +*/ diff --git a/database_test.go b/database_ex_test.go similarity index 82% rename from database_test.go rename to database_ex_test.go index 5a828bc..002ccf8 100644 --- a/database_test.go +++ b/database_ex_test.go @@ -2,7 +2,7 @@ package mdb import "time" -func (db *Database) waitForWAL() { +func (db *Database) WaitForWAL() { for { status := db.WALStatus() if status.MaxSeqNumWAL == status.MaxSeqNumKV { diff --git a/itemmap_ex_test.go b/itemmap_ex_test.go new file mode 100644 index 0000000..1e92dfa --- /dev/null +++ b/itemmap_ex_test.go @@ -0,0 +1,54 @@ +package mdb + +import ( + "fmt" + "reflect" +) + +func (m *itemMap[T]) Equals(rhs *itemMap[T]) error { + return m.EqualsMap(rhs.m) +} + +func (m *itemMap[T]) EqualsMap(data map[uint64]*T) error { + if len(data) != len(m.m) { + return fmt.Errorf("Expected %d items, but found %d.", len(data), len(m.m)) + } + + for key, exp := range data { + val, ok := m.m[key] + if !ok { + return fmt.Errorf("No value for %d. Expected: %v", key, *exp) + } + if !reflect.DeepEqual(*val, *exp) { + return fmt.Errorf("Value mismatch %d: %v != %v", key, *val, *exp) + } + } + return nil +} + +func (m *itemMap[T]) EqualsKV() (err error) { + count := 0 + m.kv.Iterate(m.collection, func(id uint64, data []byte) { + count++ + if err != nil { + return + } + item := decode[T](data) + val, ok := m.m[id] + if !ok { + err = fmt.Errorf("Item %d not found in memory: %v", id, *item) + return + } + + if !reflect.DeepEqual(*item, *val) { + err = fmt.Errorf("Items not equal %d: %v != %v", id, *item, *val) + return + } + }) + + if err == nil && count != len(m.m) { + err = fmt.Errorf("%d items on disk, but %d in memory", count, len(m.m)) + } + + return err +} diff --git a/itemmap_test.go b/itemmap_test.go new file mode 100644 index 0000000..a933509 --- /dev/null +++ b/itemmap_test.go @@ -0,0 +1,129 @@ +package mdb + +import ( + "fmt" + "reflect" + "testing" +) + +func TestItemMap(t *testing.T) { + + // expected is a map of users. + run := func(name string, inner func(t *testing.T, db *DB) (expected map[uint64]*User)) { + testWithDB(t, name, func(t *testing.T, db *DB) { + expected := inner(t, db) + + if err := db.Users.c.items.EqualsMap(expected); err != nil { + t.Fatal(err) + } + + db.WaitForWAL() + + if err := db.Users.c.items.EqualsKV(); err != nil { + t.Fatal(err) + } + + db.Close() + db = OpenDB(db.root, true) + + if err := db.Users.c.items.EqualsMap(expected); err != nil { + t.Fatal(err) + } + + if err := db.Users.c.items.EqualsKV(); err != nil { + t.Fatal(err) + } + }) + } + + run("simple", func(t *testing.T, db *DB) (expected map[uint64]*User) { + users := map[uint64]*User{} + c := db.Users.c + for i := uint64(1); i < 10; i++ { + id := c.NextID() + users[id] = &User{ + ID: id, + Email: fmt.Sprintf("a.%d@c.com", i), + Name: fmt.Sprintf("name.%d", i), + ExtID: fmt.Sprintf("EXTID.%d", i), + } + user, err := c.Insert(*users[id]) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(user, *users[id]) { + t.Fatal(user, *users[id]) + } + } + + return users + }) + + run("insert and delete", func(t *testing.T, db *DB) (expected map[uint64]*User) { + users := map[uint64]*User{} + c := db.Users.c + for x := uint64(1); x < 10; x++ { + id := c.NextID() + users[id] = &User{ + ID: id, + Email: fmt.Sprintf("a.%d@c.com", x), + Name: fmt.Sprintf("name.%d", x), + ExtID: fmt.Sprintf("EXTID.%d", x), + } + user, err := c.Insert(*users[id]) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(user, *users[id]) { + t.Fatal(user, *users[id]) + } + } + + var id uint64 + for key := range users { + id = key + } + + delete(users, id) + c.Delete(id) + + return users + }) + + run("update", func(t *testing.T, db *DB) (expected map[uint64]*User) { + users := map[uint64]*User{} + c := db.Users.c + for x := uint64(1); x < 10; x++ { + id := c.NextID() + users[id] = &User{ + ID: id, + Email: fmt.Sprintf("a.%d@c.com", x), + Name: fmt.Sprintf("name.%d", x), + ExtID: fmt.Sprintf("EXTID.%d", x), + } + user, err := c.Insert(*users[id]) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(user, *users[id]) { + t.Fatal(user, *users[id]) + } + } + + var id uint64 + for key := range users { + id = key + } + + err := c.Update(id, func(u User) (User, error) { + u.Name = "Hello" + return u, nil + }) + if err != nil { + t.Fatal(err) + } + users[id].Name = "Hello" + + return users + }) +} diff --git a/main_test.go b/main_test.go index 0002bea..89e8d55 100644 --- a/main_test.go +++ b/main_test.go @@ -11,3 +11,17 @@ func TestMain(m *testing.M) { rand.Seed(time.Now().UnixNano()) os.Exit(m.Run()) } + +func testWithDB(t *testing.T, name string, inner func(t *testing.T, db *DB)) { + t.Run(name, func(t *testing.T) { + root, err := os.MkdirTemp("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(root) + + db := OpenDB(root, true) + defer db.Close() + inner(t, db) + }) +} diff --git a/mapindex_ex_test.go b/mapindex_ex_test.go new file mode 100644 index 0000000..79a0e53 --- /dev/null +++ b/mapindex_ex_test.go @@ -0,0 +1,28 @@ +package mdb + +import ( + "fmt" + "reflect" +) + +func (m *MapIndex[K, T]) Equals(rhs *MapIndex[K, T]) error { + return m.EqualsMap(rhs.m) +} + +func (m *MapIndex[K, T]) EqualsMap(data map[K]*T) error { + if len(m.m) != len(data) { + return fmt.Errorf("Expected %d items, but found %d.", len(data), len(m.m)) + } + + for key, exp := range data { + val, ok := m.m[key] + if !ok { + return fmt.Errorf("No value for %v. Expected: %v", key, *exp) + } + if !reflect.DeepEqual(*val, *exp) { + return fmt.Errorf("Value mismatch %v: %v != %v", key, *val, *exp) + } + } + + return nil +} diff --git a/mapindex_test.go b/mapindex_test.go index d78ce91..e82704d 100644 --- a/mapindex_test.go +++ b/mapindex_test.go @@ -1,14 +1,6 @@ package mdb -import ( - "errors" - "fmt" - "os" - "path/filepath" - "reflect" - "testing" -) - +/* func TestMapIndex(t *testing.T) { type Item struct { ID uint64 @@ -328,3 +320,4 @@ func TestMapIndexLoadError(t *testing.T) { t.Fatal(err) } } +*/ diff --git a/shipping_test.go b/shipping_test.go index 18563e6..5d64c1e 100644 --- a/shipping_test.go +++ b/shipping_test.go @@ -1,12 +1,6 @@ package mdb -import ( - "net" - "os" - "path/filepath" - "testing" -) - +/* func TestLogShip(t *testing.T) { type Item struct { ID uint64 @@ -79,3 +73,4 @@ func TestLogShip(t *testing.T) { } } +*/ diff --git a/testdb_test.go b/testdb_test.go new file mode 100644 index 0000000..529e14e --- /dev/null +++ b/testdb_test.go @@ -0,0 +1,187 @@ +package mdb + +import ( + "errors" + "net/mail" + "strings" + "time" +) + +// ---------------------------------------------------------------------------- +// Validate errors. +// ---------------------------------------------------------------------------- + +var ErrInvalidName = errors.New("invalid name") + +// ---------------------------------------------------------------------------- +// User Collection +// ---------------------------------------------------------------------------- + +type User struct { + ID uint64 + Email string + Name string + ExtID string +} + +type Users struct { + c *Collection[User] + emailMap *MapIndex[string, User] // Full map index. + emailBTree *BTreeIndex[User] // Full btree index. + nameBTree *BTreeIndex[User] // Full btree with duplicates. + extIDMap *MapIndex[string, User] // Partial map index. + extIDBTree *BTreeIndex[User] // Partial btree index. +} + +func userGetID(u *User) uint64 { return u.ID } + +func userSanitize(u *User) { + u.Name = strings.TrimSpace(u.Name) + e, err := mail.ParseAddress(strings.ToLower(strings.TrimSpace(u.Email))) + if err == nil { + u.Email = e.Address + } +} + +func userValidate(u *User) error { + if len(u.Name) == 0 { + return ErrInvalidName + } + return nil +} + +// ---------------------------------------------------------------------------- +// Account Collection +// ---------------------------------------------------------------------------- + +type Account struct { + ID uint64 + Name string +} + +type Accounts struct { + c *Collection[Account] + nameMap *MapIndex[string, Account] +} + +func accountGetID(a *Account) uint64 { return a.ID } + +// ---------------------------------------------------------------------------- +// Database +// ---------------------------------------------------------------------------- + +type DB struct { + *Database + root string + Users Users + Accounts Accounts +} + +func OpenDB(root string, primary bool) *DB { + db := &DB{root: root} + if primary { + db.Database = NewPrimary(root) + } else { + db.Database = NewSecondary(root) + } + + db.Users = Users{} + db.Users.c = NewCollection(db.Database, "users", userGetID) + db.Users.c.SetSanitize(userSanitize) + db.Users.c.SetValidate(userValidate) + + db.Users.emailMap = NewMapIndex( + db.Users.c, + "email", + func(u *User) string { return u.Email }, + nil) + + db.Users.emailBTree = NewBTreeIndex( + db.Users.c, + func(lhs, rhs *User) bool { return lhs.Email < rhs.Email }, + nil) + + db.Users.nameBTree = NewBTreeIndex( + db.Users.c, + func(lhs, rhs *User) bool { + if lhs.Name != rhs.Name { + return lhs.Name < rhs.Name + } + return lhs.ID < rhs.ID + }, + nil) + + db.Users.extIDMap = NewMapIndex( + db.Users.c, + "extID", + func(u *User) string { return u.ExtID }, + func(u *User) bool { return u.ExtID != "" }) + + db.Users.extIDBTree = NewBTreeIndex( + db.Users.c, + func(lhs, rhs *User) bool { return lhs.ExtID < rhs.ExtID }, + func(u *User) bool { return u.ExtID != "" }) + + db.Accounts = Accounts{} + db.Accounts.c = NewCollection(db.Database, "accounts", accountGetID) + + return db +} + +func (db *DB) Equals(rhs *DB) error { + db.WaitForSync(rhs) + + // Users: itemMap. + if err := db.Users.c.items.Equals(rhs.Users.c.items); err != nil { + return err + } + + // Users: emailMap + if err := db.Users.emailMap.Equals(rhs.Users.emailMap); err != nil { + return err + } + + // Users: emailBTree + if err := db.Users.emailBTree.Equals(rhs.Users.emailBTree); err != nil { + return err + } + + // Users: nameBTree + if err := db.Users.nameBTree.Equals(rhs.Users.nameBTree); err != nil { + return err + } + + // Users: extIDMap + if err := db.Users.extIDMap.Equals(rhs.Users.extIDMap); err != nil { + return err + } + + // Users: extIDBTree + if err := db.Users.extIDBTree.Equals(rhs.Users.extIDBTree); err != nil { + return err + } + + // Accounts: itemMap + if err := db.Accounts.c.items.Equals(rhs.Accounts.c.items); err != nil { + return err + } + + // Accounts: nameMap + if err := db.Accounts.nameMap.Equals(rhs.Accounts.nameMap); err != nil { + return err + } + + return nil +} + +// Wait for two databases to become synchronized. +func (db *DB) WaitForSync(rhs *DB) { + for { + s1 := db.WALStatus() + s2 := rhs.WALStatus() + if s1 == s2 && s1.MaxSeqNumKV == s1.MaxSeqNumWAL { + return + } + time.Sleep(100 * time.Millisecond) + } +}