mdb/kvstore/wal/writer_test.go

package wal

import (
	"bytes"
	"fmt"
	"os"
	"strconv"
	"sync"
	"testing"
	"time"
)

// ----------------------------------------------------------------------------

func (w *Writer) waitForSeqNum(n uint64) {
	for {
		if w.MaxSeqNum() == n {
			return
		}
		time.Sleep(time.Millisecond)
	}
}

// ----------------------------------------------------------------------------

func TestWriter(t *testing.T) {

	run := func(name string, inner func(t *testing.T, walPath string, w *Writer)) {
		t.Run(name, func(t *testing.T) {
			walPath := randPath() + ".wal"
			defer os.RemoveAll(walPath)
			w := newWriter(walPath, true)
			defer w.Close()
			inner(t, walPath, w)
		})
	}

	run("simple", func(t *testing.T, walPath string, w *Writer) {
		w.Store("a", 1, _b("Hello"))
		w.Delete("b", 1)
		w.Store("a", 2, _b("World"))
		w.Store("a", 1, _b("Good bye"))

		err := walEqual(walPath, []Record{
			{SeqNum: 1, Collection: "a", ID: 1, Store: true, Data: _b("Hello")},
			{SeqNum: 2, Collection: "b", ID: 1},
			{SeqNum: 3, Collection: "a", ID: 2, Store: true, Data: _b("World")},
			{SeqNum: 4, Collection: "a", ID: 1, Store: true, Data: _b("Good bye")},
		})

		if err != nil {
			t.Fatal(err)
		}
	})

	run("write close write", func(t *testing.T, walPath string, w *Writer) {
		w.Store("a", 1, _b("Hello"))
		w.Close()

		w = newWriter(walPath, true)
		w.Delete("b", 1)
		w.Close()

		w = newWriter(walPath, true)
		w.Store("a", 2, _b("World"))
		w.Close()

		w = newWriter(walPath, true)
		w.Store("a", 1, _b("Good bye"))

		err := walEqual(walPath, []Record{
			{SeqNum: 1, Collection: "a", ID: 1, Store: true, Data: _b("Hello")},
			{SeqNum: 2, Collection: "b", ID: 1},
			{SeqNum: 3, Collection: "a", ID: 2, Store: true, Data: _b("World")},
			{SeqNum: 4, Collection: "a", ID: 1, Store: true, Data: _b("Good bye")},
		})

		if err != nil {
			t.Fatal(err)
		}
	})

	run("write concurrent", func(t *testing.T, walPath string, w *Writer) {
		N := 32
		wg := sync.WaitGroup{}

		expected := make([][]Record, N)

		for i := 0; i < N; i++ {
			wg.Add(1)
			go func(i int) {
				defer wg.Done()
				collection := fmt.Sprintf("%d", i)
				for j := 0; j < 1024; j++ {
					rec := Record{
						Collection: collection,
						ID:         uint64(j + 1),
						Store:      true,
						Data:       _b(fmt.Sprintf("%d", j)),
					}

					w.Store(rec.Collection, rec.ID, rec.Data)
					expected[i] = append(expected[i], rec)
				}
			}(i)
		}

		wg.Wait()

		recs := readWAL(walPath)
		found := make([][]Record, N)
		for _, rec := range recs {
			rec := rec
			index, err := strconv.ParseInt(rec.Collection, 10, 64)
			if err != nil {
				t.Fatal(err)
			}
			found[index] = append(found[index], rec)
		}

		if len(found) != len(expected) {
			t.Fatal(len(found), len(expected))
		}

		for i := range found {
			fnd := found[i]
			exp := expected[i]
			if len(fnd) != len(exp) {
				t.Fatal(i, len(fnd), len(exp))
			}

			for j := range fnd {
				f := fnd[j]
				e := exp[j]
				ok := f.Collection == e.Collection &&
					f.ID == e.ID &&
					f.Store == e.Store &&
					bytes.Equal(f.Data, e.Data)
				if !ok {
					t.Fatal(i, j, f, e)
				}
			}
		}
	})

	run("store delete async", func(t *testing.T, walPath string, w *Writer) {
		w.storeAsync("a", 1, _b("hello1"))
		w.storeAsync("a", 2, _b("hello2"))
		w.deleteAsync("a", 1)
		w.storeAsync("a", 3, _b("hello3"))
		w.storeAsync("b", 1, _b("b1"))

		w.waitForSeqNum(5)

		err := walEqual(walPath, []Record{
			{SeqNum: 1, Collection: "a", ID: 1, Store: true, Data: _b("hello1")},
			{SeqNum: 2, Collection: "a", ID: 2, Store: true, Data: _b("hello2")},
			{SeqNum: 3, Collection: "a", ID: 1, Store: false},
			{SeqNum: 4, Collection: "a", ID: 3, Store: true, Data: _b("hello3")},
			{SeqNum: 5, Collection: "b", ID: 1, Store: true, Data: _b("b1")},
		})

		if err != nil {
			t.Fatal(err)
		}
	})

	run("store delete async with close", func(t *testing.T, walPath string, w *Writer) {
		w.storeAsync("a", 1, _b("hello1"))
		w.Close()
		w = newWriter(walPath, true)
		w.storeAsync("a", 2, _b("hello2"))
		w.Close()
		w = newWriter(walPath, true)
		w.deleteAsync("a", 1)
		w.Close()
		w = newWriter(walPath, true)
		w.storeAsync("a", 3, _b("hello3"))
		w.Close()
		w = newWriter(walPath, true)
		w.storeAsync("b", 1, _b("b1"))
		w.Close()
		w = newWriter(walPath, true)

		w.waitForSeqNum(5)

		err := walEqual(walPath, []Record{
			{SeqNum: 1, Collection: "a", ID: 1, Store: true, Data: _b("hello1")},
			{SeqNum: 2, Collection: "a", ID: 2, Store: true, Data: _b("hello2")},
			{SeqNum: 3, Collection: "a", ID: 1, Store: false},
			{SeqNum: 4, Collection: "a", ID: 3, Store: true, Data: _b("hello3")},
			{SeqNum: 5, Collection: "b", ID: 1, Store: true, Data: _b("b1")},
		})

		if err != nil {
			t.Fatal(err)
		}
	})

	// This is really just a benchmark.
	run("store async many", func(t *testing.T, walPath string, w *Writer) {
		N := 32768
		for i := 0; i < N; i++ {
			w.storeAsync("a", 1, _b("x"))
		}
		w.waitForSeqNum(uint64(N))
	})
}