From f7cbd9fca232b0cc8fbb66e68ae9d19264f9f946 Mon Sep 17 00:00:00 2001 From: jdl Date: Tue, 26 Jul 2022 23:55:29 +0200 Subject: [PATCH] wip: testing --- collection.go | 2 - itemmap.go | 2 +- mapindex_ex_test.go | 6 +- mapindex_test.go | 440 ++++++++++++++++---------------------------- testdb_test.go | 2 + 5 files changed, 162 insertions(+), 290 deletions(-) diff --git a/collection.go b/collection.go index 43dd7e3..554e775 100644 --- a/collection.go +++ b/collection.go @@ -2,7 +2,6 @@ package mdb import ( "fmt" - "log" "git.crumpington.com/private/mdb/keyedmutex" ) @@ -179,7 +178,6 @@ func (c *Collection[T]) loadData() { toRemove := []int{} for i, idx := range c.indices { if err := idx.load(c.items.m); err != nil { - log.Printf("Removing index %d because of error: %v", i, err) toRemove = append([]int{i}, toRemove...) } } diff --git a/itemmap.go b/itemmap.go index 1d561f6..06fdcbc 100644 --- a/itemmap.go +++ b/itemmap.go @@ -101,6 +101,6 @@ func (idx *itemMap[T]) mapGet(id uint64) (*T, bool) { // ---------------------------------------------------------------------------- func (idx *itemMap[T]) nextID() uint64 { - n := rand.Int63n(256) + n := 1 + rand.Int63n(256) return atomic.AddUint64(&idx.maxID, uint64(n)) } diff --git a/mapindex_ex_test.go b/mapindex_ex_test.go index 79a0e53..f0726f4 100644 --- a/mapindex_ex_test.go +++ b/mapindex_ex_test.go @@ -15,12 +15,12 @@ func (m *MapIndex[K, T]) EqualsMap(data map[K]*T) error { } for key, exp := range data { - val, ok := m.m[key] + val, ok := m.Get(key) if !ok { return fmt.Errorf("No value for %v. Expected: %v", key, *exp) } - if !reflect.DeepEqual(*val, *exp) { - return fmt.Errorf("Value mismatch %v: %v != %v", key, *val, *exp) + if !reflect.DeepEqual(val, *exp) { + return fmt.Errorf("Value mismatch %v: %v != %v", key, val, *exp) } } diff --git a/mapindex_test.go b/mapindex_test.go index e82704d..f1bd643 100644 --- a/mapindex_test.go +++ b/mapindex_test.go @@ -1,323 +1,195 @@ package mdb -/* -func TestMapIndex(t *testing.T) { - type Item struct { - ID uint64 - Name string - ExtID string - } +import ( + "fmt" + "reflect" + "testing" +) - checkIdxOne := func(idx *MapIndex[string, Item], expectedList ...Item) error { - expected := make(map[string]Item, len(expectedList)) - for _, i := range expectedList { - expected[i.ExtID] = i - } +func TestFullMapIndex(t *testing.T) { - if len(expected) != len(idx.m) { - return fmt.Errorf("Expected %d items, but got %d.", len(expected), len(idx.m)) - } + // Test against the emailMap index. + run := func(name string, inner func(t *testing.T, db *DB) map[string]*User) { + testWithDB(t, name, func(t *testing.T, db *DB) { + expected := inner(t, db) - for _, e := range expected { - i, ok := idx.Get(e.ExtID) - if !ok { - return fmt.Errorf("Missing item: %v", e) + if err := db.Users.emailMap.EqualsMap(expected); err != nil { + t.Fatal(err) } - if !reflect.DeepEqual(i, e) { - return fmt.Errorf("Items not equal: %v != %v", i, e) + + db.Close() + db = OpenDB(db.root, true) + + if err := db.Users.emailMap.EqualsMap(expected); err != nil { + t.Fatal(err) } - } - return nil - } - - checkIdx := func(idx *MapIndex[string, Item], expectedList ...Item) error { - idx.c.db.waitForWAL() - - if err := checkIdxOne(idx, expectedList...); err != nil { - return fmt.Errorf("%w: original", err) - } - - // Reload the database, collection, and index and re-test. - db := NewPrimary(idx.c.db.root) - c := NewCollection(db, "collection", func(i *Item) uint64 { return i.ID }) - idx = NewMapIndex(c, - "ExtID", - func(i *Item) string { return i.ExtID }, - func(i *Item) bool { return i.ExtID != "" }) - db.Start() - return checkIdxOne(idx, expectedList...) - } - - run := func(name string, inner func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item])) { - t.Run(name, func(t *testing.T) { - root := filepath.Join(os.TempDir(), randString()) - defer os.RemoveAll(root) - - db := NewPrimary(root) - defer db.Close() - c := NewCollection(db, "collection", func(i *Item) uint64 { return i.ID }) - idx := NewMapIndex(c, - "ExtID", - func(i *Item) string { return i.ExtID }, - func(i *Item) bool { return i.ExtID != "" }) - db.Start() - inner(t, c, idx) }) } - run("insert item not in index", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - item := Item{4, "4", ""} - c.Insert(item) - if err := checkIdx(idx); err != nil { - t.Fatal(err) + run("insert", func(t *testing.T, db *DB) map[string]*User { + users := map[string]*User{} + + for i := uint64(1); i < 10; i++ { + user := &User{ + ID: db.Users.c.NextID(), + Email: fmt.Sprintf("a.%d@c.com", i), + Name: fmt.Sprintf("name.%d", i), + ExtID: fmt.Sprintf("EXTID.%d", i), + } + + user2, err := db.Users.c.Insert(*user) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(*user, user2) { + t.Fatal(*user, user2) + } + users[user.Email] = user } + + return users }) - run("insert item in index", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - item1 := Item{4, "4", ""} - item2 := Item{5, "5", "abcd"} - c.Insert(item1) - c.Insert(item2) - if err := checkIdx(idx, item2); err != nil { - t.Fatal(err) + // TODO: insert duplicate + + run("delete", func(t *testing.T, db *DB) map[string]*User { + users := map[string]*User{} + + for i := uint64(1); i < 10; i++ { + user := &User{ + ID: db.Users.c.NextID(), + Email: fmt.Sprintf("a.%d@c.com", i), + Name: fmt.Sprintf("name.%d", i), + ExtID: fmt.Sprintf("EXTID.%d", i), + } + + user2, err := db.Users.c.Insert(*user) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(*user, user2) { + t.Fatal(*user, user2) + } + users[user.Email] = user } + + var id string + for key := range users { + id = key + break + } + + delete(users, id) + db.Users.emailMap.Delete(id) + + return users }) - run("insert several items", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - item1 := Item{4, "4", ""} - item2 := Item{5, "5", "abcd"} - item3 := Item{6, "6", ""} - item4 := Item{7, "7", "xyz"} - item5 := Item{8, "8", ""} - item6 := Item{9, "9", "mmm"} - c.Insert(item1) - c.Insert(item2) - c.Insert(item3) - c.Insert(item4) - c.Insert(item5) - c.Insert(item6) - if err := checkIdx(idx, item2, item4, item6); err != nil { - t.Fatal(err) - } - }) + run("update non-indexed field", func(t *testing.T, db *DB) map[string]*User { + users := map[string]*User{} - run("insert with conflict", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - item1 := Item{1, "1", "one"} - item2 := Item{2, "2", "one"} - c.Insert(item1) - if _, err := c.Insert(item2); !errors.Is(err, ErrDuplicate) { - t.Fatal(err) - } - }) + for i := uint64(1); i < 10; i++ { + user := &User{ + ID: db.Users.c.NextID(), + Email: fmt.Sprintf("a.%d@c.com", i), + Name: fmt.Sprintf("name.%d", i), + ExtID: fmt.Sprintf("EXTID.%d", i), + } - run("update into index", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - item1 := Item{1, "1", ""} - c.Insert(item1) - - if err := checkIdx(idx); err != nil { - t.Fatal(err) + user2, err := db.Users.c.Insert(*user) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(*user, user2) { + t.Fatal(*user, user2) + } + users[user.Email] = user } - err := c.Update(1, func(i Item) (Item, error) { - i.ExtID = "xx" - return i, nil + var id string + for key := range users { + id = key + break + } + + err := db.Users.emailMap.Update(id, func(u User) (User, error) { + u.Name = "UPDATED" + return u, nil }) if err != nil { t.Fatal(err) } - item1.ExtID = "xx" - if err := checkIdx(idx, item1); err != nil { - t.Fatal(err) - } + users[id].Name = "UPDATED" + + return users }) - run("update out of index", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - c.Insert(Item{1, "1", ""}) - c.Insert(Item{2, "2", "two"}) // In index. - c.Insert(Item{3, "3", ""}) + run("update indexed field", func(t *testing.T, db *DB) map[string]*User { + users := map[string]*User{} - err := c.Update(1, func(in Item) (Item, error) { - in.Name = "ONE" - return in, nil + for i := uint64(1); i < 10; i++ { + user := &User{ + ID: db.Users.c.NextID(), + Email: fmt.Sprintf("a.%d@c.com", i), + Name: fmt.Sprintf("name.%d", i), + ExtID: fmt.Sprintf("EXTID.%d", i), + } + + user2, err := db.Users.c.Insert(*user) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(*user, user2) { + t.Fatal(*user, user2) + } + users[user.Email] = user + } + + var id uint64 + var email string + for key := range users { + email = key + id = users[key].ID + break + } + + err := db.Users.c.Update(id, func(u User) (User, error) { + u.Email = "test@x.com" + return u, nil }) if err != nil { t.Fatal(err) } - if err := checkIdx(idx, Item{2, "2", "two"}); err != nil { - t.Fatal(err) - } + user := users[email] + user.Email = "test@x.com" + delete(users, email) + users[user.Email] = user + + return users }) - run("update out of index conflict", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - c.Insert(Item{1, "1", ""}) - c.Insert(Item{2, "2", "two"}) // In index. - c.Insert(Item{3, "3", ""}) + // update index field change key error - err := c.Update(1, func(in Item) (Item, error) { - in.ExtID = "two" - return in, nil - }) - if !errors.Is(err, ErrDuplicate) { - t.Fatal(err) - } - }) - - run("update within index", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - c.Insert(Item{1, "1", "one"}) - c.Insert(Item{2, "2", "two"}) // In index. - c.Insert(Item{3, "3", ""}) - - err := c.Update(2, func(in Item) (Item, error) { - in.ExtID = "TWO" - return in, nil - }) - if err != nil { - t.Fatal(err) - } - - if err := checkIdx(idx, Item{1, "1", "one"}, Item{2, "2", "TWO"}); err != nil { - t.Fatal(err) - } - }) - - run("update using index", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - c.Insert(Item{1, "1", "one"}) - c.Insert(Item{2, "2", "two"}) // In index. - c.Insert(Item{3, "3", ""}) - - err := idx.Update("one", func(in Item) (Item, error) { - in.Name = "_1_" - return in, nil - }) - if err != nil { - t.Fatal(err) - } - - if err := checkIdx(idx, Item{1, "_1_", "one"}, Item{2, "2", "two"}); err != nil { - t.Fatal(err) - } - }) - - run("update using index not found", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - c.Insert(Item{1, "1", "one"}) - c.Insert(Item{3, "3", ""}) - - err := idx.Update("onex", func(in Item) (Item, error) { - in.Name = "_1_" - return in, nil - }) - if !errors.Is(err, ErrNotFound) { - t.Fatal(err) - } - }) - - run("update using index caller error", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - c.Insert(Item{1, "1", "one"}) - c.Insert(Item{3, "3", ""}) - - myErr := errors.New("Mine") - - err := idx.Update("one", func(in Item) (Item, error) { - in.Name = "_1_" - return in, myErr - }) - if !errors.Is(err, myErr) { - t.Fatal(err) - } - }) - - run("update using index mismatched IDs", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - c.Insert(Item{1, "1", "one"}) - - err := idx.Update("one", func(in Item) (Item, error) { - in.ExtID = "onex" - return in, nil - }) - if !errors.Is(err, ErrMismatchedIDs) { - t.Fatal(err) - } - }) - - run("delete out of index", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - c.Insert(Item{1, "1", "one"}) - c.Insert(Item{2, "2", "two"}) // In index. - c.Insert(Item{3, "3", ""}) - c.Delete(3) - - if err := checkIdx(idx, Item{1, "1", "one"}, Item{2, "2", "two"}); err != nil { - t.Fatal(err) - } - }) - - run("delete from index", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - c.Insert(Item{1, "1", "one"}) - c.Insert(Item{2, "2", "two"}) // In index. - c.Insert(Item{3, "3", ""}) - c.Delete(2) - - if err := checkIdx(idx, Item{1, "1", "one"}); err != nil { - t.Fatal(err) - } - }) - - run("delete using index", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - c.Insert(Item{1, "1", "one"}) - c.Insert(Item{2, "2", "two"}) // In index. - c.Insert(Item{3, "3", ""}) - idx.Delete("two") - - if err := checkIdx(idx, Item{1, "1", "one"}); err != nil { - t.Fatal(err) - } - }) - - run("delete using index not found", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - c.Insert(Item{1, "1", "one"}) - c.Insert(Item{2, "2", "two"}) // In index. - c.Insert(Item{3, "3", ""}) - idx.Delete("onex") - - if err := checkIdx(idx, Item{1, "1", "one"}, Item{2, "2", "two"}); err != nil { - t.Fatal(err) - } - }) - - run("check name", func(t *testing.T, c *Collection[Item], idx *MapIndex[string, Item]) { - if idx.name() != "ExtID" { - t.Fatal(idx.name()) - } - }) + // update error from func + // update w/ ErrAbortUpdate + // update not found. + // update conflict. } -func TestMapIndexLoadError(t *testing.T) { - type Item struct { - ID uint64 - Name string - ExtID string - } - - root := filepath.Join(os.TempDir(), randString()) - defer os.RemoveAll(root) - - db := NewPrimary(root) - c := NewCollection(db, "collection", func(i *Item) uint64 { return i.ID }) - db.Start() - defer db.Close() - - c.Insert(Item{1, "one", "x"}) - c.Insert(Item{2, "two", "x"}) - c.Insert(Item{3, "three", "y"}) - c.Insert(Item{4, "x", ""}) - - idx := NewMapIndex(c, - "ExtID", - func(i *Item) string { return i.ExtID }, - func(i *Item) bool { return i.ExtID != "" }) - err := idx.load(c.items.m) - if !errors.Is(err, ErrDuplicate) { - t.Fatal(err) - } +func TestPartialMapIndex(t *testing.T) { + // insert into index + // insert into index conflict + // insert outside index + // insert and delete in index + // insert and delete outside index + // insert withing index + // update out of index + // update into index + // update error from func + // udpate ErrAbortUpdate + // update not found + // update conflict in to in + // update conflict out to in } -*/ diff --git a/testdb_test.go b/testdb_test.go index 529e14e..ac162eb 100644 --- a/testdb_test.go +++ b/testdb_test.go @@ -125,6 +125,8 @@ func OpenDB(root string, primary bool) *DB { db.Accounts = Accounts{} db.Accounts.c = NewCollection(db.Database, "accounts", accountGetID) + db.Start() + return db }