diff --git a/fstore/browser.go b/fstore/browser.go index bfe71fb..4643ecf 100644 --- a/fstore/browser.go +++ b/fstore/browser.go @@ -3,10 +3,11 @@ package fstore import ( "embed" "io" - "git.crumpington.com/public/jldb/fstore/pages" "net/http" "os" "path/filepath" + + "git.crumpington.com/public/jldb/fstore/pages" ) //go:embed static/* diff --git a/fstore/command.go b/fstore/command.go index 07ffa81..8d58241 100644 --- a/fstore/command.go +++ b/fstore/command.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "io" + "git.crumpington.com/public/jldb/lib/errs" ) diff --git a/fstore/paths.go b/fstore/paths.go index 0efe19a..46c1cc6 100644 --- a/fstore/paths.go +++ b/fstore/paths.go @@ -1,9 +1,10 @@ package fstore import ( - "git.crumpington.com/public/jldb/lib/errs" "path/filepath" "strconv" + + "git.crumpington.com/public/jldb/lib/errs" ) func filesRootPath(rootDir string) string { diff --git a/fstore/store-commands.go b/fstore/store-commands.go index 6889e2a..6710ca1 100644 --- a/fstore/store-commands.go +++ b/fstore/store-commands.go @@ -1,11 +1,12 @@ package fstore import ( - "git.crumpington.com/public/jldb/lib/errs" - "git.crumpington.com/public/jldb/lib/idgen" "log" "os" "path/filepath" + + "git.crumpington.com/public/jldb/lib/errs" + "git.crumpington.com/public/jldb/lib/idgen" ) func (s *Store) applyStoreFromReader(cmd command) error { diff --git a/fstore/store-rep.go b/fstore/store-rep.go index 4d3d563..ea1c426 100644 --- a/fstore/store-rep.go +++ b/fstore/store-rep.go @@ -5,12 +5,13 @@ import ( "errors" "io" "io/fs" - "git.crumpington.com/public/jldb/lib/errs" - "git.crumpington.com/public/jldb/lib/wal" "net" "os" "path/filepath" "time" + + "git.crumpington.com/public/jldb/lib/errs" + "git.crumpington.com/public/jldb/lib/wal" ) func (s *Store) repSendState(conn net.Conn) error { diff --git a/fstore/store.go b/fstore/store.go index e2d13de..7676129 100644 --- a/fstore/store.go +++ b/fstore/store.go @@ -3,15 +3,16 @@ package fstore import ( "bytes" "io" - "git.crumpington.com/public/jldb/lib/errs" - "git.crumpington.com/public/jldb/lib/idgen" - "git.crumpington.com/public/jldb/lib/rep" "net/http" "os" "path/filepath" "strconv" "sync" "time" + + "git.crumpington.com/public/jldb/lib/errs" + "git.crumpington.com/public/jldb/lib/idgen" + "git.crumpington.com/public/jldb/lib/rep" ) type Config struct { diff --git a/lib/atomicheader/atomicheader.go b/lib/atomicheader/atomicheader.go index 9404dab..4396b93 100644 --- a/lib/atomicheader/atomicheader.go +++ b/lib/atomicheader/atomicheader.go @@ -3,9 +3,10 @@ package atomicheader import ( "encoding/binary" "hash/crc32" - "git.crumpington.com/public/jldb/lib/errs" "os" "sync" + + "git.crumpington.com/public/jldb/lib/errs" ) const ( diff --git a/lib/httpconn/client.go b/lib/httpconn/client.go index 39e6935..d5b7d2b 100644 --- a/lib/httpconn/client.go +++ b/lib/httpconn/client.go @@ -6,11 +6,12 @@ import ( "crypto/tls" "errors" "io" - "git.crumpington.com/public/jldb/lib/errs" "net" "net/http" "net/url" "time" + + "git.crumpington.com/public/jldb/lib/errs" ) var ErrInvalidStatus = errors.New("invalid status") diff --git a/lib/rep/functions.go b/lib/rep/functions.go deleted file mode 100644 index d9ac3d2..0000000 --- a/lib/rep/functions.go +++ /dev/null @@ -1,51 +0,0 @@ -package rep - -import ( - "encoding/binary" - "encoding/json" - "git.crumpington.com/public/jldb/lib/errs" - "net" - "path/filepath" - "time" -) - -// ---------------------------------------------------------------------------- - -func lockFilePath(rootDir string) string { - return filepath.Join(rootDir, "lock") -} - -func walRootDir(rootDir string) string { - return filepath.Join(rootDir, "wal") -} - -func stateFilePath(rootDir string) string { - return filepath.Join(rootDir, "state") -} - -// ---------------------------------------------------------------------------- - -func sendJSON( - item any, - conn net.Conn, - timeout time.Duration, -) error { - - buf := bufPoolGet() - defer bufPoolPut(buf) - - if err := json.NewEncoder(buf).Encode(item); err != nil { - return errs.Unexpected.WithErr(err) - } - - sizeBuf := make([]byte, 2) - binary.LittleEndian.PutUint16(sizeBuf, uint16(buf.Len())) - - conn.SetWriteDeadline(time.Now().Add(timeout)) - buffers := net.Buffers{sizeBuf, buf.Bytes()} - if _, err := buffers.WriteTo(conn); err != nil { - return errs.IO.WithErr(err) - } - - return nil -} diff --git a/lib/rep/http-client.go b/lib/rep/http-client.go index a5e4fe2..4b3fdbf 100644 --- a/lib/rep/http-client.go +++ b/lib/rep/http-client.go @@ -1,178 +1,109 @@ package rep import ( - "encoding/binary" "encoding/json" - "io" + "net" + "net/http" + "strings" + "sync" + "time" + "git.crumpington.com/public/jldb/lib/errs" "git.crumpington.com/public/jldb/lib/httpconn" "git.crumpington.com/public/jldb/lib/wal" - "net" - "sync" - "time" ) type client struct { - // Mutex-protected variables. - lock sync.Mutex - closed bool - conn net.Conn + client *http.Client // The following are constant. endpoint string - psk []byte + pskBytes [64]byte timeout time.Duration - buf []byte + lock sync.Mutex + conn net.Conn } func newClient(endpoint, psk string, timeout time.Duration) *client { - b := make([]byte, 256) - copy(b, []byte(psk)) + httpClient := &http.Client{ + Timeout: timeout, + } + + if !strings.HasSuffix(endpoint, "/") { + endpoint += "/" + } return &client{ + client: httpClient, endpoint: endpoint, - psk: b, + pskBytes: pskToBytes(psk), timeout: timeout, } } func (c *client) GetInfo() (info Info, err error) { - err = c.withConn(cmdGetInfo, func(conn net.Conn) error { - return c.recvJSON(&info, conn, c.timeout) - }) - return info, err + req, err := http.NewRequest(http.MethodGet, c.endpoint+pathGetInfo, nil) + if err != nil { + return info, errs.Unexpected.WithErr(err) + } + req.SetBasicAuth("", string(c.pskBytes[:])) + + resp, err := c.client.Do(req) + if err != nil { + return info, errs.IO.WithErr(err) + } + defer resp.Body.Close() + + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + return info, errs.IO.WithErr(err) + } + return info, nil } func (c *client) RecvState(recv func(net.Conn) error) error { - return c.withConn(cmdSendState, recv) + err := c.dialConnect(c.endpoint + pathSendState) + if err != nil { + return err + } + defer c.conn.Close() + + return recv(c.conn) } func (c *client) StreamWAL(w *wal.WAL) error { - return c.withConn(cmdStreamWAL, func(conn net.Conn) error { - return w.Recv(conn, c.timeout) - }) + err := c.dialConnect(c.endpoint + pathStreamWAL) + if err != nil { + return err + } + defer c.conn.Close() + + return w.Recv(c.conn, c.timeout) } func (c *client) Close() { c.lock.Lock() defer c.lock.Unlock() - c.closed = true - if c.conn != nil { c.conn.Close() - c.conn = nil } } // ---------------------------------------------------------------------------- -func (c *client) writeCmd(cmd byte) error { - c.conn.SetWriteDeadline(time.Now().Add(c.timeout)) - if _, err := c.conn.Write([]byte{cmd}); err != nil { - return errs.IO.WithErr(err) - } - return nil -} - -func (c *client) dial() error { - c.conn = nil - - conn, err := httpconn.Dial(c.endpoint) +func (c *client) dialConnect(endpoint string) error { + conn, err := httpconn.Dial(endpoint) if err != nil { return err } conn.SetWriteDeadline(time.Now().Add(c.timeout)) - if _, err := conn.Write(c.psk); err != nil { - conn.Close() + if _, err := conn.Write(c.pskBytes[:]); err != nil { return errs.IO.WithErr(err) } + c.lock.Lock() + defer c.lock.Unlock() c.conn = conn return nil } - -func (c *client) withConn(cmd byte, fn func(net.Conn) error) error { - conn, err := c.getConn(cmd) - if err != nil { - return err - } - - if err := fn(conn); err != nil { - conn.Close() - return err - } - return nil -} - -func (c *client) getConn(cmd byte) (net.Conn, error) { - c.lock.Lock() - defer c.lock.Unlock() - - if c.closed { - return nil, errs.IO.WithErr(io.EOF) - } - - dialed := false - - if c.conn == nil { - if err := c.dial(); err != nil { - return nil, err - } - dialed = true - } - - if err := c.writeCmd(cmd); err != nil { - if dialed { - c.conn = nil - return nil, err - } - - if err := c.dial(); err != nil { - return nil, err - } - - if err := c.writeCmd(cmd); err != nil { - return nil, err - } - } - - return c.conn, nil -} - -func (c *client) recvJSON( - item any, - conn net.Conn, - timeout time.Duration, -) error { - - if cap(c.buf) < 2 { - c.buf = make([]byte, 0, 1024) - } - buf := c.buf[:2] - - conn.SetReadDeadline(time.Now().Add(timeout)) - - if _, err := io.ReadFull(conn, buf); err != nil { - return errs.IO.WithErr(err) - } - - size := binary.LittleEndian.Uint16(buf) - - if cap(buf) < int(size) { - buf = make([]byte, size) - c.buf = buf - } - buf = buf[:size] - - if _, err := io.ReadFull(conn, buf); err != nil { - return errs.IO.WithErr(err) - } - - if err := json.Unmarshal(buf, item); err != nil { - return errs.Unexpected.WithErr(err) - } - - return nil -} diff --git a/lib/rep/http-handler-util.go b/lib/rep/http-handler-util.go new file mode 100644 index 0000000..05d4877 --- /dev/null +++ b/lib/rep/http-handler-util.go @@ -0,0 +1,54 @@ +package rep + +import ( + "crypto/subtle" + "io" + "log" + "net" + "net/http" + "time" + + "git.crumpington.com/public/jldb/lib/httpconn" +) + +func (rep *Replicator) handlerLogf(pattern string, args ...any) { + log.Printf("[HTTP-HANDLER] "+pattern, args...) +} + +// checkBasicAuth tests whether the caller has provided the appropriate basic auth +// header. The caller should provide the PSK as the basic-auth password. +func (rep *Replicator) checkBasicAuth(w http.ResponseWriter, r *http.Request) bool { + _, pwd, _ := r.BasicAuth() + if subtle.ConstantTimeCompare([]byte(pwd), rep.pskBytes[:]) != 1 { + rep.handlerLogf("PSK mismatch.") + http.Error(w, "not authorized", http.StatusUnauthorized) + return false + } + return true +} + +// acceptConnect accepts a CONNECT request and checks the PSK. +func (rep *Replicator) acceptConnect(w http.ResponseWriter, r *http.Request) net.Conn { + conn, err := httpconn.Accept(w, r) + if err != nil { + rep.handlerLogf("Failed to accept connection: %s", err) + return nil + } + + psk := [64]byte{} + + conn.SetReadDeadline(time.Now().Add(rep.conf.NetTimeout)) + if _, err := io.ReadFull(conn, psk[:]); err != nil { + conn.Close() + rep.handlerLogf("Failed to read PSK: %v", err) + return nil + } + + if subtle.ConstantTimeCompare(psk[:], rep.pskBytes[:]) != 1 { + conn.Close() + rep.handlerLogf("PSK mismatch.") + return nil + } + + return conn +} diff --git a/lib/rep/http-handler.go b/lib/rep/http-handler.go index fe513e4..843cb7a 100644 --- a/lib/rep/http-handler.go +++ b/lib/rep/http-handler.go @@ -1,79 +1,69 @@ package rep import ( - "crypto/subtle" - "git.crumpington.com/public/jldb/lib/httpconn" - "log" + "encoding/json" "net/http" - "time" + "path" ) const ( - cmdGetInfo = 10 - cmdSendState = 20 - cmdStreamWAL = 30 + pathGetInfo = "get-info" + pathSendState = "send-state" + pathStreamWAL = "stream-wal" ) -// --------------------------------------------------------------------------- - func (rep *Replicator) Handle(w http.ResponseWriter, r *http.Request) { - logf := func(pattern string, args ...any) { - log.Printf("[HTTP-HANDLER] "+pattern, args...) + // We'll handle two types of requests: HTTP GET requests for JSON, or + // streaming requets for state or wall. + + base := path.Base(r.URL.Path) + switch base { + case pathGetInfo: + rep.handleGetInfo(w, r) + case pathSendState: + rep.handleSendState(w, r) + case pathStreamWAL: + rep.handleStreamWAL(w, r) + default: + http.Error(w, "not found", http.StatusNotFound) + } +} + +func (rep *Replicator) handleGetInfo(w http.ResponseWriter, r *http.Request) { + if !rep.checkBasicAuth(w, r) { + return } - conn, err := httpconn.Accept(w, r) - if err != nil { - logf("Failed to accept connection: %s", err) + w.Header().Set("Content-Type", "application/json") + + if err := json.NewEncoder(w).Encode(rep.Info()); err != nil { + rep.handlerLogf("Failed to send info: %s", err) + } +} + +func (rep *Replicator) handleSendState(w http.ResponseWriter, r *http.Request) { + conn := rep.acceptConnect(w, r) + if conn == nil { return } defer conn.Close() - psk := make([]byte, 256) - - conn.SetReadDeadline(time.Now().Add(rep.conf.NetTimeout)) - if _, err := conn.Read(psk); err != nil { - logf("Failed to read PSK: %v", err) - return - } - - expected := rep.pskBytes - if subtle.ConstantTimeCompare(expected, psk) != 1 { - logf("PSK mismatch.") - return - } - - cmd := make([]byte, 1) - - for { - conn.SetReadDeadline(time.Now().Add(rep.conf.NetTimeout)) - if _, err := conn.Read(cmd); err != nil { - logf("Read failed: %v", err) - return - } - - switch cmd[0] { - - case cmdGetInfo: - if err := sendJSON(rep.Info(), conn, rep.conf.NetTimeout); err != nil { - logf("Failed to send info: %s", err) - return - } - - case cmdSendState: - - if err := rep.sendState(conn); err != nil { - if !rep.stopped() { - logf("Failed to send state: %s", err) - } - return - } - - case cmdStreamWAL: - err := rep.wal.Send(conn, rep.conf.NetTimeout) - if !rep.stopped() { - logf("Failed when sending WAL: %s", err) - } - return + if err := rep.sendState(conn); err != nil { + if !rep.stopped() { + rep.handlerLogf("Failed to send state: %s", err) } } } + +func (rep *Replicator) handleStreamWAL(w http.ResponseWriter, r *http.Request) { + conn := rep.acceptConnect(w, r) + if conn == nil { + return + } + defer conn.Close() + + err := rep.wal.Send(conn, rep.conf.NetTimeout) + if !rep.stopped() { + rep.handlerLogf("Failed when streaming WAL: %s", err) + } +} diff --git a/lib/rep/paths.go b/lib/rep/paths.go new file mode 100644 index 0000000..4241832 --- /dev/null +++ b/lib/rep/paths.go @@ -0,0 +1,17 @@ +package rep + +import ( + "path/filepath" +) + +func lockFilePath(rootDir string) string { + return filepath.Join(rootDir, "lock") +} + +func walRootDir(rootDir string) string { + return filepath.Join(rootDir, "wal") +} + +func stateFilePath(rootDir string) string { + return filepath.Join(rootDir, "state") +} diff --git a/lib/rep/pools.go b/lib/rep/pools.go deleted file mode 100644 index e539223..0000000 --- a/lib/rep/pools.go +++ /dev/null @@ -1,21 +0,0 @@ -package rep - -import ( - "bytes" - "sync" -) - -var bufPool = sync.Pool{ - New: func() any { - return &bytes.Buffer{} - }, -} - -func bufPoolGet() *bytes.Buffer { - return bufPool.Get().(*bytes.Buffer) -} - -func bufPoolPut(b *bytes.Buffer) { - b.Reset() - bufPool.Put(b) -} diff --git a/lib/rep/psk.go b/lib/rep/psk.go new file mode 100644 index 0000000..617ac2a --- /dev/null +++ b/lib/rep/psk.go @@ -0,0 +1,16 @@ +package rep + +import ( + "crypto/sha256" + "encoding/hex" +) + +func pskToBytes(in string) [64]byte { + b := sha256.Sum256([]byte(in)) + dst := [64]byte{} + i := hex.Encode(dst[:], b[:]) + if i != 64 { + panic(i) + } + return dst +} diff --git a/lib/rep/rep-sendrecv.go b/lib/rep/rep-sendrecv.go index 184b49a..c4b72ba 100644 --- a/lib/rep/rep-sendrecv.go +++ b/lib/rep/rep-sendrecv.go @@ -2,9 +2,10 @@ package rep import ( "io" - "git.crumpington.com/public/jldb/lib/errs" "net" "time" + + "git.crumpington.com/public/jldb/lib/errs" ) func (rep *Replicator) sendState(conn net.Conn) error { diff --git a/lib/rep/replicator-open.go b/lib/rep/replicator-open.go index 0eb6b04..57f1a5e 100644 --- a/lib/rep/replicator-open.go +++ b/lib/rep/replicator-open.go @@ -1,12 +1,13 @@ package rep import ( + "os" + "time" + "git.crumpington.com/public/jldb/lib/atomicheader" "git.crumpington.com/public/jldb/lib/errs" "git.crumpington.com/public/jldb/lib/flock" "git.crumpington.com/public/jldb/lib/wal" - "os" - "time" ) func (rep *Replicator) loadConfigDefaults() { @@ -26,9 +27,7 @@ func (rep *Replicator) loadConfigDefaults() { } rep.conf = conf - - rep.pskBytes = make([]byte, 256) - copy(rep.pskBytes, []byte(conf.ReplicationPSK)) + rep.pskBytes = pskToBytes(conf.ReplicationPSK) } func (rep *Replicator) initDirectories() error { diff --git a/lib/rep/replicator-walrecvr.go b/lib/rep/replicator-walrecvr.go index a5b5b05..039282e 100644 --- a/lib/rep/replicator-walrecvr.go +++ b/lib/rep/replicator-walrecvr.go @@ -30,9 +30,8 @@ func (rep *Replicator) runWALRecvrOnce() { log.Printf("[WAL-RECVR] "+pattern, args...) } - if err := rep.client.StreamWAL(rep.wal); err != nil { - if !rep.stopped() { - logf("Recv failed: %v", err) - } + err := rep.client.StreamWAL(rep.wal) + if !rep.stopped() { + logf("Recv failed: %v", err) } } diff --git a/lib/rep/replicator.go b/lib/rep/replicator.go index 2e4ce6e..43bda0b 100644 --- a/lib/rep/replicator.go +++ b/lib/rep/replicator.go @@ -2,14 +2,15 @@ package rep import ( "io" - "git.crumpington.com/public/jldb/lib/atomicheader" - "git.crumpington.com/public/jldb/lib/errs" - "git.crumpington.com/public/jldb/lib/wal" "net" "os" "sync" "sync/atomic" "time" + + "git.crumpington.com/public/jldb/lib/atomicheader" + "git.crumpington.com/public/jldb/lib/errs" + "git.crumpington.com/public/jldb/lib/wal" ) type Config struct { @@ -59,8 +60,9 @@ type Replicator struct { conf Config lockFile *os.File - pskBytes []byte - wal *wal.WAL + pskBytes [64]byte // 64 ascii characters. See pskToBytes. + + wal *wal.WAL appendNotify chan struct{} @@ -161,10 +163,6 @@ func (rep *Replicator) Primary() bool { return rep.conf.Primary } -// TODO: Probably remove this. -// The caller may call Ack after Apply to acknowledge that the change has also -// been applied to the caller's application. Alternatively, the caller may use -// follow to apply changes to their application state. func (rep *Replicator) ack(seqNum, timestampMS int64) error { state := rep.getState() state.SeqNum = seqNum diff --git a/lib/rep/testapp_test.go b/lib/rep/testapp_test.go index 6438e12..163b7c6 100644 --- a/lib/rep/testapp_test.go +++ b/lib/rep/testapp_test.go @@ -5,12 +5,14 @@ import ( "encoding/binary" "encoding/json" "io" - "git.crumpington.com/public/jldb/lib/wal" "math/rand" "net" "sync" "testing" "time" + + "git.crumpington.com/public/jldb/lib/errs" + "git.crumpington.com/public/jldb/lib/wal" ) // ---------------------------------------------------------------------------- @@ -82,7 +84,7 @@ func newApp(t *testing.T, id int64, conf Config) *TestApp { Apply: a.apply, }, conf) if err != nil { - t.Fatal(err) + t.Fatal(errs.FmtDetails(err)) } return a diff --git a/lib/wal/corrupt_test.go b/lib/wal/corrupt_test.go index 82f5021..d5b77f7 100644 --- a/lib/wal/corrupt_test.go +++ b/lib/wal/corrupt_test.go @@ -2,8 +2,9 @@ package wal import ( "io" - "git.crumpington.com/public/jldb/lib/errs" "testing" + + "git.crumpington.com/public/jldb/lib/errs" ) func TestCorruptWAL(t *testing.T) { diff --git a/lib/wal/generic_test.go b/lib/wal/generic_test.go index 4ee6ba2..6a0f4d7 100644 --- a/lib/wal/generic_test.go +++ b/lib/wal/generic_test.go @@ -5,13 +5,14 @@ import ( "encoding/binary" "errors" "io" - "git.crumpington.com/public/jldb/lib/errs" "math/rand" "path/filepath" "reflect" "strings" "testing" "time" + + "git.crumpington.com/public/jldb/lib/errs" ) type waLog interface { diff --git a/lib/wal/io.go b/lib/wal/io.go index b042973..b7400e7 100644 --- a/lib/wal/io.go +++ b/lib/wal/io.go @@ -5,6 +5,7 @@ import ( "errors" "hash/crc32" "io" + "git.crumpington.com/public/jldb/lib/errs" ) diff --git a/lib/wal/record.go b/lib/wal/record.go index baf41c9..db6914a 100644 --- a/lib/wal/record.go +++ b/lib/wal/record.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "hash/crc32" "io" + "git.crumpington.com/public/jldb/lib/errs" ) diff --git a/lib/wal/record_test.go b/lib/wal/record_test.go index 365411c..e4faffe 100644 --- a/lib/wal/record_test.go +++ b/lib/wal/record_test.go @@ -3,10 +3,11 @@ package wal import ( "bytes" "io" - "git.crumpington.com/public/jldb/lib/errs" - "git.crumpington.com/public/jldb/lib/testutil" "math/rand" "testing" + + "git.crumpington.com/public/jldb/lib/errs" + "git.crumpington.com/public/jldb/lib/testutil" ) func NewRecordForTesting() Record { diff --git a/lib/wal/segment-iterator.go b/lib/wal/segment-iterator.go index 6495f2b..890732d 100644 --- a/lib/wal/segment-iterator.go +++ b/lib/wal/segment-iterator.go @@ -1,10 +1,11 @@ package wal import ( - "git.crumpington.com/public/jldb/lib/atomicheader" - "git.crumpington.com/public/jldb/lib/errs" "os" "time" + + "git.crumpington.com/public/jldb/lib/atomicheader" + "git.crumpington.com/public/jldb/lib/errs" ) type segmentIterator struct { diff --git a/lib/wal/segment.go b/lib/wal/segment.go index c4bfaa4..44daaee 100644 --- a/lib/wal/segment.go +++ b/lib/wal/segment.go @@ -3,11 +3,12 @@ package wal import ( "bufio" "io" - "git.crumpington.com/public/jldb/lib/atomicheader" - "git.crumpington.com/public/jldb/lib/errs" "os" "sync" "time" + + "git.crumpington.com/public/jldb/lib/atomicheader" + "git.crumpington.com/public/jldb/lib/errs" ) type segment struct { diff --git a/lib/wal/segment_test.go b/lib/wal/segment_test.go index ab922f7..20b8078 100644 --- a/lib/wal/segment_test.go +++ b/lib/wal/segment_test.go @@ -4,11 +4,12 @@ import ( "bytes" crand "crypto/rand" "io" - "git.crumpington.com/public/jldb/lib/atomicheader" - "git.crumpington.com/public/jldb/lib/errs" "path/filepath" "testing" "time" + + "git.crumpington.com/public/jldb/lib/atomicheader" + "git.crumpington.com/public/jldb/lib/errs" ) func newSegmentForTesting(t *testing.T) *segment { diff --git a/lib/wal/wal-iterator.go b/lib/wal/wal-iterator.go index 5d0902d..0da5d43 100644 --- a/lib/wal/wal-iterator.go +++ b/lib/wal/wal-iterator.go @@ -1,8 +1,9 @@ package wal import ( - "git.crumpington.com/public/jldb/lib/errs" "time" + + "git.crumpington.com/public/jldb/lib/errs" ) type walIterator struct { diff --git a/lib/wal/wal-recv.go b/lib/wal/wal-recv.go index 60cf008..eb991b8 100644 --- a/lib/wal/wal-recv.go +++ b/lib/wal/wal-recv.go @@ -3,9 +3,10 @@ package wal import ( "encoding/binary" "io" - "git.crumpington.com/public/jldb/lib/errs" "net" "time" + + "git.crumpington.com/public/jldb/lib/errs" ) func (wal *WAL) Recv(conn net.Conn, timeout time.Duration) error { diff --git a/lib/wal/wal-send.go b/lib/wal/wal-send.go index 7ab7e70..51688b0 100644 --- a/lib/wal/wal-send.go +++ b/lib/wal/wal-send.go @@ -2,9 +2,10 @@ package wal import ( "encoding/binary" - "git.crumpington.com/public/jldb/lib/errs" "net" "time" + + "git.crumpington.com/public/jldb/lib/errs" ) const ( diff --git a/lib/wal/wal-sendrecv_test.go b/lib/wal/wal-sendrecv_test.go index e4d2100..7fda037 100644 --- a/lib/wal/wal-sendrecv_test.go +++ b/lib/wal/wal-sendrecv_test.go @@ -1,8 +1,6 @@ package wal import ( - "git.crumpington.com/public/jldb/lib/errs" - "git.crumpington.com/public/jldb/lib/testutil" "log" "math/rand" "reflect" @@ -11,6 +9,9 @@ import ( "sync/atomic" "testing" "time" + + "git.crumpington.com/public/jldb/lib/errs" + "git.crumpington.com/public/jldb/lib/testutil" ) func TestSendRecvHarness(t *testing.T) { diff --git a/lib/wal/wal.go b/lib/wal/wal.go index adeea7d..cff400b 100644 --- a/lib/wal/wal.go +++ b/lib/wal/wal.go @@ -2,13 +2,14 @@ package wal import ( "io" - "git.crumpington.com/public/jldb/lib/atomicheader" - "git.crumpington.com/public/jldb/lib/errs" "os" "path/filepath" "strconv" "sync" "time" + + "git.crumpington.com/public/jldb/lib/atomicheader" + "git.crumpington.com/public/jldb/lib/errs" ) type Config struct { diff --git a/mdb/change/binary.go b/mdb/change/binary.go index 727ce20..c7ae196 100644 --- a/mdb/change/binary.go +++ b/mdb/change/binary.go @@ -3,6 +3,7 @@ package change import ( "encoding/binary" "io" + "git.crumpington.com/public/jldb/lib/errs" ) diff --git a/mdb/change/change.go b/mdb/change/change.go index ff8f07d..415cf16 100644 --- a/mdb/change/change.go +++ b/mdb/change/change.go @@ -2,6 +2,7 @@ package change import ( "io" + "git.crumpington.com/public/jldb/lib/errs" ) diff --git a/mdb/collection.go b/mdb/collection.go index cd4bf63..73b035d 100644 --- a/mdb/collection.go +++ b/mdb/collection.go @@ -5,9 +5,10 @@ import ( "encoding/json" "errors" "hash/crc64" - "git.crumpington.com/public/jldb/lib/errs" "unsafe" + "git.crumpington.com/public/jldb/lib/errs" + "github.com/google/btree" ) @@ -20,10 +21,10 @@ type Collection[T any] struct { sanitize func(*T) validate func(*T) error - indices []Index[T] - uniqueIndices []Index[T] + indices []*Index[T] + uniqueIndices []*Index[T] - ByID Index[T] + ByID *Index[T] buf *bytes.Buffer } @@ -64,8 +65,8 @@ func NewCollection[T any](db *Database, name string, conf *CollectionConfig[T]) copy: conf.Copy, sanitize: conf.Sanitize, validate: conf.Validate, - indices: []Index[T]{}, - uniqueIndices: []Index[T]{}, + indices: []*Index[T]{}, + uniqueIndices: []*Index[T]{}, buf: &bytes.Buffer{}, } @@ -91,7 +92,7 @@ func NewCollection[T any](db *Database, name string, conf *CollectionConfig[T]) return c } -func (c Collection[T]) Name() string { +func (c *Collection[T]) Name() string { return c.name } @@ -107,13 +108,13 @@ type indexConfig[T any] struct { Include func(item *T) bool } -func (c Collection[T]) Get(tx *Snapshot, id uint64) (*T, bool) { +func (c *Collection[T]) Get(tx *Snapshot, id uint64) (*T, bool) { x := new(T) c.setID(x, id) return c.ByID.Get(tx, x) } -func (c Collection[T]) List(tx *Snapshot, ids []uint64, out []*T) []*T { +func (c *Collection[T]) List(tx *Snapshot, ids []uint64, out []*T) []*T { if len(ids) == 0 { return out[:0] } @@ -134,8 +135,7 @@ func (c Collection[T]) List(tx *Snapshot, ids []uint64, out []*T) []*T { } // AddIndex: Add an index to the collection. -func (c *Collection[T]) addIndex(conf indexConfig[T]) Index[T] { - +func (c *Collection[T]) addIndex(conf indexConfig[T]) *Index[T] { var less func(*T, *T) bool if conf.Unique { @@ -159,7 +159,7 @@ func (c *Collection[T]) addIndex(conf indexConfig[T]) Index[T] { BTree: btree.NewG(256, less), } - index := Index[T]{ + index := &Index[T]{ collectionID: c.collectionID, name: conf.Name, indexID: c.getState(c.db.Snapshot()).addIndex(indexState), @@ -175,7 +175,7 @@ func (c *Collection[T]) addIndex(conf indexConfig[T]) Index[T] { return index } -func (c Collection[T]) Insert(tx *Snapshot, userItem *T) error { +func (c *Collection[T]) Insert(tx *Snapshot, userItem *T) error { if err := c.ensureMutable(tx); err != nil { return err } @@ -189,7 +189,7 @@ func (c Collection[T]) Insert(tx *Snapshot, userItem *T) error { for i := range c.uniqueIndices { if c.uniqueIndices[i].insertConflict(tx, item) { - return ErrDuplicate.WithCollection(c.name).WithIndex(c.uniqueIndices[i].name) + return errs.Duplicate.WithCollection(c.name).WithIndex(c.uniqueIndices[i].name) } } @@ -202,7 +202,7 @@ func (c Collection[T]) Insert(tx *Snapshot, userItem *T) error { return nil } -func (c Collection[T]) Update(tx *Snapshot, userItem *T) error { +func (c *Collection[T]) Update(tx *Snapshot, userItem *T) error { if err := c.ensureMutable(tx); err != nil { return err } @@ -216,12 +216,12 @@ func (c Collection[T]) Update(tx *Snapshot, userItem *T) error { old, ok := c.ByID.get(tx, item) if !ok { - return ErrNotFound + return errs.NotFound } for i := range c.uniqueIndices { if c.uniqueIndices[i].updateConflict(tx, item) { - return ErrDuplicate.WithCollection(c.name).WithIndex(c.uniqueIndices[i].name) + return errs.Duplicate.WithCollection(c.name).WithIndex(c.uniqueIndices[i].name) } } @@ -234,18 +234,18 @@ func (c Collection[T]) Update(tx *Snapshot, userItem *T) error { return nil } -func (c Collection[T]) Upsert(tx *Snapshot, item *T) error { +func (c *Collection[T]) Upsert(tx *Snapshot, item *T) error { err := c.Insert(tx, item) if err == nil { return nil } - if errors.Is(err, ErrDuplicate) { + if errors.Is(err, errs.Duplicate) { return c.Update(tx, item) } return err } -func (c Collection[T]) Delete(tx *Snapshot, itemID uint64) error { +func (c *Collection[T]) Delete(tx *Snapshot, itemID uint64) error { if err := c.ensureMutable(tx); err != nil { return err } @@ -253,15 +253,15 @@ func (c Collection[T]) Delete(tx *Snapshot, itemID uint64) error { return c.deleteItem(tx, itemID) } -func (c Collection[T]) getByID(tx *Snapshot, itemID uint64) (*T, bool) { +func (c *Collection[T]) getByID(tx *Snapshot, itemID uint64) (*T, bool) { x := new(T) c.setID(x, itemID) return c.ByID.get(tx, x) } -func (c Collection[T]) ensureMutable(tx *Snapshot) error { +func (c *Collection[T]) ensureMutable(tx *Snapshot) error { if !tx.writable() { - return ErrReadOnly + return errs.ReadOnly } state := c.getState(tx) @@ -273,7 +273,7 @@ func (c Collection[T]) ensureMutable(tx *Snapshot) error { } // For initial data loading. -func (c Collection[T]) insertItem(tx *Snapshot, itemID uint64, data []byte) error { +func (c *Collection[T]) insertItem(tx *Snapshot, itemID uint64, data []byte) error { item := new(T) if err := json.Unmarshal(data, item); err != nil { return errs.Encoding.WithErr(err).WithCollection(c.name) @@ -282,7 +282,7 @@ func (c Collection[T]) insertItem(tx *Snapshot, itemID uint64, data []byte) erro // Check for insert conflict. for _, index := range c.uniqueIndices { if index.insertConflict(tx, item) { - return ErrDuplicate + return errs.Duplicate } } @@ -294,10 +294,10 @@ func (c Collection[T]) insertItem(tx *Snapshot, itemID uint64, data []byte) erro return nil } -func (c Collection[T]) deleteItem(tx *Snapshot, itemID uint64) error { +func (c *Collection[T]) deleteItem(tx *Snapshot, itemID uint64) error { item, ok := c.getByID(tx, itemID) if !ok { - return ErrNotFound + return errs.NotFound } tx.delete(c.collectionID, itemID) @@ -311,7 +311,7 @@ func (c Collection[T]) deleteItem(tx *Snapshot, itemID uint64) error { // upsertItem inserts or updates the item with itemID and the given serialized // form. It's called by -func (c Collection[T]) upsertItem(tx *Snapshot, itemID uint64, data []byte) error { +func (c *Collection[T]) upsertItem(tx *Snapshot, itemID uint64, data []byte) error { item, ok := c.getByID(tx, itemID) if ok { tx.delete(c.collectionID, itemID) @@ -334,14 +334,14 @@ func (c Collection[T]) upsertItem(tx *Snapshot, itemID uint64, data []byte) erro return nil } -func (c Collection[T]) getID(t *T) uint64 { +func (c *Collection[T]) getID(t *T) uint64 { return *((*uint64)(unsafe.Pointer(t))) } -func (c Collection[T]) setID(t *T, id uint64) { +func (c *Collection[T]) setID(t *T, id uint64) { *((*uint64)(unsafe.Pointer(t))) = id } -func (c Collection[T]) getState(tx *Snapshot) *collectionState[T] { +func (c *Collection[T]) getState(tx *Snapshot) *collectionState[T] { return tx.collections[c.collectionID].(*collectionState[T]) } diff --git a/mdb/crashconsistency_test.go b/mdb/crashconsistency_test.go index ead50f8..67bccfc 100644 --- a/mdb/crashconsistency_test.go +++ b/mdb/crashconsistency_test.go @@ -1,12 +1,13 @@ package mdb import ( - "git.crumpington.com/public/jldb/lib/errs" "log" "os" "os/exec" "testing" "time" + + "git.crumpington.com/public/jldb/lib/errs" ) func TestCrashConsistency(t *testing.T) { diff --git a/mdb/db-primary.go b/mdb/db-primary.go deleted file mode 100644 index df6ba5b..0000000 --- a/mdb/db-primary.go +++ /dev/null @@ -1,67 +0,0 @@ -package mdb - -/* -func (db *Database) openPrimary() (err error) { - wal, err := cwal.Open(db.walRootDir, cwal.Config{ - SegMinCount: db.conf.WALSegMinCount, - SegMaxAgeSec: db.conf.WALSegMaxAgeSec, - }) - - pFile, err := pfile.Open(db.pageFilePath, - - pFile, err := openPageFileAndReplayWAL(db.rootDir) - if err != nil { - return err - } - defer pFile.Close() - - pfHeader, err := pFile.ReadHeader() - if err != nil { - return err - } - - tx := db.Snapshot() - tx.seqNum = pfHeader.SeqNum - tx.updatedAt = pfHeader.UpdatedAt - - pIndex, err := pagefile.NewIndex(pFile) - if err != nil { - return err - } - - err = pFile.IterateAllocated(pIndex, func(cID, iID uint64, data []byte) error { - return db.loadItem(tx, cID, iID, data) - }) - if err != nil { - return err - } - - w, err := cwal.OpenWriter(db.walRootDir, &cwal.WriterConfig{ - SegMinCount: db.conf.WALSegMinCount, - SegMaxAgeSec: db.conf.WALSegMaxAgeSec, - }) - if err != nil { - return err - } - - db.done.Add(1) - go txAggregator{ - Stop: db.stop, - Done: db.done, - ModChan: db.modChan, - W: w, - Index: pIndex, - Snapshot: db.snapshot, - }.Run() - - db.done.Add(1) - go (&fileWriter{ - Stop: db.stop, - Done: db.done, - PageFilePath: db.pageFilePath, - WALRootDir: db.walRootDir, - }).Run() - - return nil -} -*/ diff --git a/mdb/db-rep.go b/mdb/db-rep.go index fef2ba2..70dd3eb 100644 --- a/mdb/db-rep.go +++ b/mdb/db-rep.go @@ -1,13 +1,14 @@ package mdb import ( + "log" + "net" + "os" + "git.crumpington.com/public/jldb/lib/errs" "git.crumpington.com/public/jldb/lib/wal" "git.crumpington.com/public/jldb/mdb/change" "git.crumpington.com/public/jldb/mdb/pfile" - "log" - "net" - "os" ) func (db *Database) repSendState(conn net.Conn) error { diff --git a/mdb/db-secondary.go b/mdb/db-secondary.go deleted file mode 100644 index f03b01f..0000000 --- a/mdb/db-secondary.go +++ /dev/null @@ -1,129 +0,0 @@ -package mdb - -/* -func (db *Database) openSecondary() (err error) { - if db.shouldLoadFromPrimary() { - if err := db.loadFromPrimary(); err != nil { - return err - } - } - - log.Printf("Opening page-file...") - - pFile, err := openPageFileAndReplayWAL(db.rootDir) - if err != nil { - return err - } - defer pFile.Close() - - pfHeader, err := pFile.ReadHeader() - if err != nil { - return err - } - - log.Printf("Building page-file index...") - - pIndex, err := pagefile.NewIndex(pFile) - if err != nil { - return err - } - - tx := db.Snapshot() - tx.seqNum = pfHeader.SeqNum - tx.updatedAt = pfHeader.UpdatedAt - - log.Printf("Loading data into memory...") - - err = pFile.IterateAllocated(pIndex, func(cID, iID uint64, data []byte) error { - return db.loadItem(tx, cID, iID, data) - }) - if err != nil { - return err - } - - log.Printf("Creating writer...") - - w, err := cswal.OpenWriter(db.walRootDir, &cswal.WriterConfig{ - SegMinCount: db.conf.WALSegMinCount, - SegMaxAgeSec: db.conf.WALSegMaxAgeSec, - }) - if err != nil { - return err - } - - db.done.Add(1) - go (&walFollower{ - Stop: db.stop, - Done: db.done, - W: w, - Client: NewClient(db.conf.PrimaryURL, db.conf.ReplicationPSK, db.conf.NetTimeout), - }).Run() - - db.done.Add(1) - go (&follower{ - Stop: db.stop, - Done: db.done, - WALRootDir: db.walRootDir, - SeqNum: pfHeader.SeqNum, - ApplyChanges: db.applyChanges, - }).Run() - - db.done.Add(1) - go (&fileWriter{ - Stop: db.stop, - Done: db.done, - PageFilePath: db.pageFilePath, - WALRootDir: db.walRootDir, - }).Run() - - return nil -} - -func (db *Database) shouldLoadFromPrimary() bool { - if _, err := os.Stat(db.walRootDir); os.IsNotExist(err) { - log.Printf("WAL doesn't exist.") - return true - } - if _, err := os.Stat(db.pageFilePath); os.IsNotExist(err) { - log.Printf("Page-file doesn't exist.") - return true - } - return false -} - -func (db *Database) loadFromPrimary() error { - client := NewClient(db.conf.PrimaryURL, db.conf.ReplicationPSK, db.conf.NetTimeout) - defer client.Disconnect() - - log.Printf("Loading data from primary...") - - if err := os.RemoveAll(db.pageFilePath); err != nil { - log.Printf("Failed to remove page-file: %s", err) - return errs.IO.WithErr(err) // Caller can retry. - } - - if err := os.RemoveAll(db.walRootDir); err != nil { - log.Printf("Failed to remove WAL: %s", err) - return errs.IO.WithErr(err) // Caller can retry. - } - - err := client.DownloadPageFile(db.pageFilePath+".tmp", db.pageFilePath) - if err != nil { - log.Printf("Failed to get page-file from primary: %s", err) - return err // Caller can retry. - } - - pfHeader, err := pagefile.ReadHeader(db.pageFilePath) - if err != nil { - log.Printf("Failed to read page-file sequence number: %s", err) - return err // Caller can retry. - } - - if err = cswal.CreateEx(db.walRootDir, pfHeader.SeqNum+1); err != nil { - log.Printf("Failed to initialize WAL: %s", err) - return err // Caller can retry. - } - - return nil -} -*/ diff --git a/mdb/db-testcases_test.go b/mdb/db-testcases_test.go index 626d4ba..c05f958 100644 --- a/mdb/db-testcases_test.go +++ b/mdb/db-testcases_test.go @@ -6,6 +6,8 @@ import ( "reflect" "strings" "testing" + + "git.crumpington.com/public/jldb/lib/errs" ) type DBTestCase struct { @@ -54,7 +56,7 @@ var testDBTestCases = []DBTestCase{{ Update: func(t *testing.T, db TestDB, tx *Snapshot) error { user, ok := db.Users.ByID.Get(tx, &User{ID: 1}) if !ok { - return ErrNotFound + return errs.NotFound } user.Name = "Bob" user.Email = "b@c.com" @@ -111,7 +113,7 @@ var testDBTestCases = []DBTestCase{{ return db.Users.Insert(tx, user2) }, - ExpectedUpdateError: ErrDuplicate, + ExpectedUpdateError: errs.Duplicate, State: DBState{}, }}, @@ -131,7 +133,7 @@ var testDBTestCases = []DBTestCase{{ return db.Users.Insert(tx, user2) }, - ExpectedUpdateError: ErrDuplicate, + ExpectedUpdateError: errs.Duplicate, State: DBState{}, }}, @@ -162,7 +164,7 @@ var testDBTestCases = []DBTestCase{{ return db.Users.Insert(tx, user) }, - ExpectedUpdateError: ErrDuplicate, + ExpectedUpdateError: errs.Duplicate, State: DBState{ UsersByID: []User{{ID: 1, Name: "Alice", Email: "a@b.com"}}, @@ -197,7 +199,7 @@ var testDBTestCases = []DBTestCase{{ return db.Users.Insert(tx, user) }, - ExpectedUpdateError: ErrDuplicate, + ExpectedUpdateError: errs.Duplicate, State: DBState{ UsersByID: []User{{ID: 1, Name: "Alice", Email: "a@b.com"}}, @@ -218,7 +220,7 @@ var testDBTestCases = []DBTestCase{{ return db.Users.Insert(db.Snapshot(), user) }, - ExpectedUpdateError: ErrReadOnly, + ExpectedUpdateError: errs.ReadOnly, }}, }, { @@ -290,7 +292,7 @@ var testDBTestCases = []DBTestCase{{ return db.Users.Update(tx, user) }, - ExpectedUpdateError: ErrNotFound, + ExpectedUpdateError: errs.NotFound, State: DBState{ UsersByID: []User{{ID: 5, Name: "Alice", Email: "a@b.com"}}, @@ -323,14 +325,14 @@ var testDBTestCases = []DBTestCase{{ Update: func(t *testing.T, db TestDB, tx *Snapshot) error { user, ok := db.Users.ByID.Get(tx, &User{ID: 1}) if !ok { - return ErrNotFound + return errs.NotFound } user.Name = "Bob" user.Email = "b@c.com" return db.Users.Update(db.Snapshot(), user) }, - ExpectedUpdateError: ErrReadOnly, + ExpectedUpdateError: errs.ReadOnly, State: DBState{ UsersByID: []User{{ID: 1, Name: "Alice", Email: "a@b.com"}}, @@ -451,7 +453,7 @@ var testDBTestCases = []DBTestCase{{ return db.Users.Update(tx, user2) }, - ExpectedUpdateError: ErrDuplicate, + ExpectedUpdateError: errs.Duplicate, State: DBState{}, }}, @@ -493,14 +495,14 @@ var testDBTestCases = []DBTestCase{{ Update: func(t *testing.T, db TestDB, tx *Snapshot) error { u, ok := db.Users.ByID.Get(tx, &User{ID: 2}) if !ok { - return ErrNotFound + return errs.NotFound } u.Email = "a@b.com" return db.Users.Update(tx, u) }, - ExpectedUpdateError: ErrDuplicate, + ExpectedUpdateError: errs.Duplicate, State: DBState{ UsersByID: []User{ @@ -542,7 +544,7 @@ var testDBTestCases = []DBTestCase{{ return db.Users.Delete(db.Snapshot(), 1) }, - ExpectedUpdateError: ErrReadOnly, + ExpectedUpdateError: errs.ReadOnly, State: DBState{ UsersByID: []User{{ID: 1, Name: "Alice", Email: "a@b.com"}}, @@ -575,7 +577,7 @@ var testDBTestCases = []DBTestCase{{ return db.Users.Delete(tx, 2) }, - ExpectedUpdateError: ErrNotFound, + ExpectedUpdateError: errs.NotFound, State: DBState{ UsersByID: []User{{ID: 1, Name: "Alice", Email: "a@b.com"}}, @@ -609,7 +611,7 @@ var testDBTestCases = []DBTestCase{{ u, ok := db.Users.ByID.Get(tx, &User{ID: 1}) if !ok { - return ErrNotFound + return errs.NotFound } if !reflect.DeepEqual(u, expected) { return errors.New("Not equal (id)") @@ -617,7 +619,7 @@ var testDBTestCases = []DBTestCase{{ u, ok = db.Users.ByEmail.Get(tx, &User{Email: "a@b.com"}) if !ok { - return ErrNotFound + return errs.NotFound } if !reflect.DeepEqual(u, expected) { return errors.New("Not equal (email)") diff --git a/mdb/db-testrunner_test.go b/mdb/db-testrunner_test.go index a812c8e..a011ffa 100644 --- a/mdb/db-testrunner_test.go +++ b/mdb/db-testrunner_test.go @@ -134,7 +134,7 @@ func checkSlicesEqual[T any](t *testing.T, name string, actual, expected []T) { } } -func checkMinMaxEqual[T any](t *testing.T, name string, tx *Snapshot, index Index[T], expected []T) { +func checkMinMaxEqual[T any](t *testing.T, name string, tx *Snapshot, index *Index[T], expected []T) { if len(expected) == 0 { if min, ok := index.Min(tx); ok { t.Fatal(min) diff --git a/mdb/db-txaggregator.go b/mdb/db-txaggregator.go index 958bd1c..9eadc9f 100644 --- a/mdb/db-txaggregator.go +++ b/mdb/db-txaggregator.go @@ -2,6 +2,7 @@ package mdb import ( "bytes" + "git.crumpington.com/public/jldb/mdb/change" ) diff --git a/mdb/db-userdata_test.go b/mdb/db-userdata_test.go index ce946fb..b6a786e 100644 --- a/mdb/db-userdata_test.go +++ b/mdb/db-userdata_test.go @@ -14,7 +14,7 @@ type UserDataItem struct { type UserData struct { *Collection[UserDataItem] - ByName Index[UserDataItem] // Unique index on (Token). + ByName *Index[UserDataItem] // Unique index on (Token). } func NewUserDataCollection(db *Database) UserData { diff --git a/mdb/db-users_test.go b/mdb/db-users_test.go index 63dbcc3..4109f2f 100644 --- a/mdb/db-users_test.go +++ b/mdb/db-users_test.go @@ -12,9 +12,9 @@ type User struct { type Users struct { *Collection[User] - ByEmail Index[User] // Unique index on (Email). - ByName Index[User] // Index on (Name). - ByBlocked Index[User] // Partial index on (Blocked,Email). + ByEmail *Index[User] // Unique index on (Email). + ByName *Index[User] // Index on (Name). + ByBlocked *Index[User] // Partial index on (Blocked,Email). } func NewUserCollection(db *Database) Users { diff --git a/mdb/db.go b/mdb/db.go index 17fe15e..a6f66ff 100644 --- a/mdb/db.go +++ b/mdb/db.go @@ -2,15 +2,16 @@ package mdb import ( "fmt" - "git.crumpington.com/public/jldb/lib/errs" - "git.crumpington.com/public/jldb/lib/rep" - "git.crumpington.com/public/jldb/mdb/change" - "git.crumpington.com/public/jldb/mdb/pfile" "net/http" "os" "sync" "sync/atomic" "time" + + "git.crumpington.com/public/jldb/lib/errs" + "git.crumpington.com/public/jldb/lib/rep" + "git.crumpington.com/public/jldb/mdb/change" + "git.crumpington.com/public/jldb/mdb/pfile" ) type Config struct { diff --git a/mdb/errors.go b/mdb/errors.go deleted file mode 100644 index 343379d..0000000 --- a/mdb/errors.go +++ /dev/null @@ -1,11 +0,0 @@ -package mdb - -import ( - "git.crumpington.com/public/jldb/lib/errs" -) - -var ( - ErrNotFound = errs.NotFound - ErrReadOnly = errs.ReadOnly - ErrDuplicate = errs.Duplicate -) diff --git a/mdb/index.go b/mdb/index.go index ed769ee..9335e64 100644 --- a/mdb/index.go +++ b/mdb/index.go @@ -10,7 +10,7 @@ func NewIndex[T any]( c *Collection[T], name string, compare func(lhs, rhs *T) int, -) Index[T] { +) *Index[T] { return c.addIndex(indexConfig[T]{ Name: name, Unique: false, @@ -24,7 +24,7 @@ func NewPartialIndex[T any]( name string, compare func(lhs, rhs *T) int, include func(*T) bool, -) Index[T] { +) *Index[T] { return c.addIndex(indexConfig[T]{ Name: name, Unique: false, @@ -37,7 +37,7 @@ func NewUniqueIndex[T any]( c *Collection[T], name string, compare func(lhs, rhs *T) int, -) Index[T] { +) *Index[T] { return c.addIndex(indexConfig[T]{ Name: name, Unique: true, @@ -51,7 +51,7 @@ func NewUniquePartialIndex[T any]( name string, compare func(lhs, rhs *T) int, include func(*T) bool, -) Index[T] { +) *Index[T] { return c.addIndex(indexConfig[T]{ Name: name, Unique: true, @@ -70,7 +70,7 @@ type Index[T any] struct { copy func(*T) *T } -func (i Index[T]) Get(tx *Snapshot, in *T) (item *T, ok bool) { +func (i *Index[T]) Get(tx *Snapshot, in *T) (item *T, ok bool) { tPtr, ok := i.get(tx, in) if !ok { return item, false @@ -78,15 +78,15 @@ func (i Index[T]) Get(tx *Snapshot, in *T) (item *T, ok bool) { return i.copy(tPtr), true } -func (i Index[T]) get(tx *Snapshot, in *T) (*T, bool) { +func (i *Index[T]) get(tx *Snapshot, in *T) (*T, bool) { return i.btree(tx).Get(in) } -func (i Index[T]) Has(tx *Snapshot, in *T) bool { +func (i *Index[T]) Has(tx *Snapshot, in *T) bool { return i.btree(tx).Has(in) } -func (i Index[T]) Min(tx *Snapshot) (item *T, ok bool) { +func (i *Index[T]) Min(tx *Snapshot) (item *T, ok bool) { tPtr, ok := i.btree(tx).Min() if !ok { return item, false @@ -94,7 +94,7 @@ func (i Index[T]) Min(tx *Snapshot) (item *T, ok bool) { return i.copy(tPtr), true } -func (i Index[T]) Max(tx *Snapshot) (item *T, ok bool) { +func (i *Index[T]) Max(tx *Snapshot) (item *T, ok bool) { tPtr, ok := i.btree(tx).Max() if !ok { return item, false @@ -102,25 +102,25 @@ func (i Index[T]) Max(tx *Snapshot) (item *T, ok bool) { return i.copy(tPtr), true } -func (i Index[T]) Ascend(tx *Snapshot, each func(*T) bool) { +func (i *Index[T]) Ascend(tx *Snapshot, each func(*T) bool) { i.btreeForIter(tx).Ascend(func(t *T) bool { return each(i.copy(t)) }) } -func (i Index[T]) AscendAfter(tx *Snapshot, after *T, each func(*T) bool) { +func (i *Index[T]) AscendAfter(tx *Snapshot, after *T, each func(*T) bool) { i.btreeForIter(tx).AscendGreaterOrEqual(after, func(t *T) bool { return each(i.copy(t)) }) } -func (i Index[T]) Descend(tx *Snapshot, each func(*T) bool) { +func (i *Index[T]) Descend(tx *Snapshot, each func(*T) bool) { i.btreeForIter(tx).Descend(func(t *T) bool { return each(i.copy(t)) }) } -func (i Index[T]) DescendAfter(tx *Snapshot, after *T, each func(*T) bool) { +func (i *Index[T]) DescendAfter(tx *Snapshot, after *T, each func(*T) bool) { i.btreeForIter(tx).DescendLessOrEqual(after, func(t *T) bool { return each(i.copy(t)) }) @@ -133,7 +133,7 @@ type ListArgs[T any] struct { Limit int // Maximum number of items to return. 0 => All. } -func (i Index[T]) List(tx *Snapshot, args ListArgs[T], out []*T) []*T { +func (i *Index[T]) List(tx *Snapshot, args ListArgs[T], out []*T) []*T { if args.Limit < 0 { return nil } @@ -176,11 +176,11 @@ func (i Index[T]) List(tx *Snapshot, args ListArgs[T], out []*T) []*T { // ---------------------------------------------------------------------------- -func (i Index[T]) insertConflict(tx *Snapshot, item *T) bool { +func (i *Index[T]) insertConflict(tx *Snapshot, item *T) bool { return i.btree(tx).Has(item) } -func (i Index[T]) updateConflict(tx *Snapshot, item *T) bool { +func (i *Index[T]) updateConflict(tx *Snapshot, item *T) bool { current, ok := i.btree(tx).Get(item) return ok && i.getID(current) != i.getID(item) } @@ -188,7 +188,7 @@ func (i Index[T]) updateConflict(tx *Snapshot, item *T) bool { // This should only be called after insertConflict. Additionally, the caller // should ensure that the index has been properly cloned for write before // writing. -func (i Index[T]) insert(tx *Snapshot, item *T) { +func (i *Index[T]) insert(tx *Snapshot, item *T) { if i.include != nil && !i.include(item) { return } @@ -196,7 +196,7 @@ func (i Index[T]) insert(tx *Snapshot, item *T) { i.btree(tx).ReplaceOrInsert(item) } -func (i Index[T]) update(tx *Snapshot, old, new *T) { +func (i *Index[T]) update(tx *Snapshot, old, new *T) { bt := i.btree(tx) bt.Delete(old) @@ -204,22 +204,22 @@ func (i Index[T]) update(tx *Snapshot, old, new *T) { i.insert(tx, new) } -func (i Index[T]) delete(tx *Snapshot, item *T) { +func (i *Index[T]) delete(tx *Snapshot, item *T) { i.btree(tx).Delete(item) } // ---------------------------------------------------------------------------- -func (i Index[T]) getState(tx *Snapshot) indexState[T] { +func (i *Index[T]) getState(tx *Snapshot) indexState[T] { return tx.collections[i.collectionID].(*collectionState[T]).Indices[i.indexID] } // Get the current btree for get/has/update/delete, etc. -func (i Index[T]) btree(tx *Snapshot) *btree.BTreeG[*T] { +func (i *Index[T]) btree(tx *Snapshot) *btree.BTreeG[*T] { return i.getState(tx).BTree } -func (i Index[T]) btreeForIter(tx *Snapshot) *btree.BTreeG[*T] { +func (i *Index[T]) btreeForIter(tx *Snapshot) *btree.BTreeG[*T] { cState := tx.collections[i.collectionID].(*collectionState[T]) bt := cState.Indices[i.indexID].BTree @@ -231,6 +231,6 @@ func (i Index[T]) btreeForIter(tx *Snapshot) *btree.BTreeG[*T] { return bt } -func (i Index[T]) getID(t *T) uint64 { +func (i *Index[T]) getID(t *T) uint64 { return *((*uint64)(unsafe.Pointer(t))) } diff --git a/mdb/pfile/change_test.go b/mdb/pfile/change_test.go index 3cd4502..922bd11 100644 --- a/mdb/pfile/change_test.go +++ b/mdb/pfile/change_test.go @@ -2,8 +2,9 @@ package pfile import ( crand "crypto/rand" - "git.crumpington.com/public/jldb/mdb/change" "math/rand" + + "git.crumpington.com/public/jldb/mdb/change" ) func randomChangeList() (changes []change.Change) { diff --git a/mdb/pfile/main_test.go b/mdb/pfile/main_test.go index a673804..3fc06b0 100644 --- a/mdb/pfile/main_test.go +++ b/mdb/pfile/main_test.go @@ -3,10 +3,11 @@ package pfile import ( "bytes" crand "crypto/rand" - "git.crumpington.com/public/jldb/lib/wal" - "git.crumpington.com/public/jldb/mdb/change" "path/filepath" "testing" + + "git.crumpington.com/public/jldb/lib/wal" + "git.crumpington.com/public/jldb/mdb/change" ) func newForTesting(t *testing.T) (*File, *Index) { diff --git a/mdb/pfile/page.go b/mdb/pfile/page.go index bc02d4f..f0d1f93 100644 --- a/mdb/pfile/page.go +++ b/mdb/pfile/page.go @@ -2,8 +2,9 @@ package pfile import ( "hash/crc32" - "git.crumpington.com/public/jldb/lib/errs" "unsafe" + + "git.crumpington.com/public/jldb/lib/errs" ) // ---------------------------------------------------------------------------- @@ -30,7 +31,7 @@ var emptyPage = func() dataPage { type pageHeader struct { CRC uint32 // IEEE CRC-32 checksum. PageType uint32 // One of the PageType* constants. - CollectionID uint64 // + CollectionID uint64 // ItemID uint64 DataSize uint64 NextPage uint64 diff --git a/mdb/pfile/page_test.go b/mdb/pfile/page_test.go index 0edd52d..8e1b46a 100644 --- a/mdb/pfile/page_test.go +++ b/mdb/pfile/page_test.go @@ -3,9 +3,10 @@ package pfile import ( "bytes" crand "crypto/rand" - "git.crumpington.com/public/jldb/lib/errs" "math/rand" "testing" + + "git.crumpington.com/public/jldb/lib/errs" ) func randomPage(t *testing.T) dataPage { diff --git a/mdb/pfile/pagefile.go b/mdb/pfile/pagefile.go index b607da3..c889e78 100644 --- a/mdb/pfile/pagefile.go +++ b/mdb/pfile/pagefile.go @@ -6,12 +6,13 @@ import ( "compress/gzip" "encoding/binary" "io" - "git.crumpington.com/public/jldb/lib/errs" - "git.crumpington.com/public/jldb/mdb/change" "net" "os" "sync" "time" + + "git.crumpington.com/public/jldb/lib/errs" + "git.crumpington.com/public/jldb/mdb/change" ) type File struct { diff --git a/mdb/pfile/record_test.go b/mdb/pfile/record_test.go index 24d0ec5..c4395ee 100644 --- a/mdb/pfile/record_test.go +++ b/mdb/pfile/record_test.go @@ -2,6 +2,7 @@ package pfile import ( "bytes" + "git.crumpington.com/public/jldb/lib/wal" "git.crumpington.com/public/jldb/mdb/change" ) diff --git a/mdb/snapshot.go b/mdb/snapshot.go index 8476f84..119d661 100644 --- a/mdb/snapshot.go +++ b/mdb/snapshot.go @@ -3,8 +3,9 @@ package mdb import ( "bytes" "encoding/json" - "git.crumpington.com/public/jldb/mdb/change" "sync/atomic" + + "git.crumpington.com/public/jldb/mdb/change" ) type Snapshot struct { diff --git a/mdb/testing/crashconsistency/main.go b/mdb/testing/crashconsistency/main.go index 42de62c..3ee0385 100644 --- a/mdb/testing/crashconsistency/main.go +++ b/mdb/testing/crashconsistency/main.go @@ -4,7 +4,6 @@ import ( "crypto/rand" "errors" "hash/crc32" - "git.crumpington.com/public/jldb/mdb" "log" mrand "math/rand" "os" @@ -13,6 +12,8 @@ import ( "sync" "sync/atomic" "time" + + "git.crumpington.com/public/jldb/mdb" ) type DataItem struct {