public
/
jldb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 17 Wiki Activity

v2 #2

Merged
johnnylee merged 4 commits from v2 into master 2023-11-17 10:06:57 +00:00
Conversation 0 Commits 4 Files Changed 56
56 changed files with 369 additions and 604 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
fstore/browser.go
View File

@ -3,10 +3,11 @@ package fstore
import (
"embed"
"io"
"git.crumpington.com/public/jldb/fstore/pages"
"net/http"
"os"
"path/filepath"
"git.crumpington.com/public/jldb/fstore/pages"
)
//go:embed static/*

1
fstore/command.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"encoding/binary"
"io"
"git.crumpington.com/public/jldb/lib/errs"
)

3
fstore/paths.go
View File

@ -1,9 +1,10 @@
package fstore
import (
"git.crumpington.com/public/jldb/lib/errs"
"path/filepath"
"strconv"
"git.crumpington.com/public/jldb/lib/errs"
)
func filesRootPath(rootDir string) string {

5
fstore/store-commands.go
View File

@ -1,11 +1,12 @@
package fstore
import (
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/idgen"
"log"
"os"
"path/filepath"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/idgen"
)
func (s *Store) applyStoreFromReader(cmd command) error {

5
fstore/store-rep.go
View File

@ -5,12 +5,13 @@ import (
"errors"
"io"
"io/fs"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/wal"
"net"
"os"
"path/filepath"
"time"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/wal"
)
func (s *Store) repSendState(conn net.Conn) error {

7
fstore/store.go
View File

@ -3,15 +3,16 @@ package fstore
import (
"bytes"
"io"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/idgen"
"git.crumpington.com/public/jldb/lib/rep"
"net/http"
"os"
"path/filepath"
"strconv"
"sync"
"time"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/idgen"
"git.crumpington.com/public/jldb/lib/rep"
)
type Config struct {

3
lib/atomicheader/atomicheader.go
View File

@ -3,9 +3,10 @@ package atomicheader
import (
"encoding/binary"
"hash/crc32"
"git.crumpington.com/public/jldb/lib/errs"
"os"
"sync"
"git.crumpington.com/public/jldb/lib/errs"
)
const (

3
lib/httpconn/client.go
View File

@ -6,11 +6,12 @@ import (
"crypto/tls"
"errors"
"io"
"git.crumpington.com/public/jldb/lib/errs"
"net"
"net/http"
"net/url"
"time"
"git.crumpington.com/public/jldb/lib/errs"
)
var ErrInvalidStatus = errors.New("invalid status")

51
lib/rep/functions.go
View File

@ -1,51 +0,0 @@
package rep
import (
"encoding/binary"
"encoding/json"
"git.crumpington.com/public/jldb/lib/errs"
"net"
"path/filepath"
"time"
)
// ----------------------------------------------------------------------------
func lockFilePath(rootDir string) string {
return filepath.Join(rootDir, "lock")
}
func walRootDir(rootDir string) string {
return filepath.Join(rootDir, "wal")
}
func stateFilePath(rootDir string) string {
return filepath.Join(rootDir, "state")
}
// ----------------------------------------------------------------------------
func sendJSON(
item any,
conn net.Conn,
timeout time.Duration,
) error {
buf := bufPoolGet()
defer bufPoolPut(buf)
if err := json.NewEncoder(buf).Encode(item); err != nil {
return errs.Unexpected.WithErr(err)
}
sizeBuf := make([]byte, 2)
binary.LittleEndian.PutUint16(sizeBuf, uint16(buf.Len()))
conn.SetWriteDeadline(time.Now().Add(timeout))
buffers := net.Buffers{sizeBuf, buf.Bytes()}
if _, err := buffers.WriteTo(conn); err != nil {
return errs.IO.WithErr(err)
}
return nil
}

177
lib/rep/http-client.go
View File

@ -1,178 +1,109 @@
package rep
import (
"encoding/binary"
"encoding/json"
"io"
"net"
"net/http"
"strings"
"sync"
"time"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/httpconn"
"git.crumpington.com/public/jldb/lib/wal"
"net"
"sync"
"time"
)
type client struct {
// Mutex-protected variables.
lock sync.Mutex
closed bool
conn net.Conn
client *http.Client
// The following are constant.
endpoint string
psk []byte
pskBytes [64]byte
timeout time.Duration
buf []byte
lock sync.Mutex
conn net.Conn
}
func newClient(endpoint, psk string, timeout time.Duration) *client {
b := make([]byte, 256)
copy(b, []byte(psk))
httpClient := &http.Client{
Timeout: timeout,
}
if !strings.HasSuffix(endpoint, "/") {
endpoint += "/"
}
return &client{
client: httpClient,
endpoint: endpoint,
psk: b,
pskBytes: pskToBytes(psk),
timeout: timeout,
}
}
func (c *client) GetInfo() (info Info, err error) {
err = c.withConn(cmdGetInfo, func(conn net.Conn) error {
return c.recvJSON(&info, conn, c.timeout)
})
return info, err
req, err := http.NewRequest(http.MethodGet, c.endpoint+pathGetInfo, nil)
if err != nil {
return info, errs.Unexpected.WithErr(err)
}
req.SetBasicAuth("", string(c.pskBytes[:]))
resp, err := c.client.Do(req)
if err != nil {
return info, errs.IO.WithErr(err)
}
defer resp.Body.Close()
if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
return info, errs.IO.WithErr(err)
}
return info, nil
}
func (c *client) RecvState(recv func(net.Conn) error) error {
return c.withConn(cmdSendState, recv)
err := c.dialConnect(c.endpoint + pathSendState)
if err != nil {
return err
}
defer c.conn.Close()
return recv(c.conn)
}
func (c *client) StreamWAL(w *wal.WAL) error {
return c.withConn(cmdStreamWAL, func(conn net.Conn) error {
return w.Recv(conn, c.timeout)
})
err := c.dialConnect(c.endpoint + pathStreamWAL)
if err != nil {
return err
}
defer c.conn.Close()
return w.Recv(c.conn, c.timeout)
}
func (c *client) Close() {
c.lock.Lock()
defer c.lock.Unlock()
c.closed = true
if c.conn != nil {
c.conn.Close()
c.conn = nil
}
}
// ----------------------------------------------------------------------------
func (c *client) writeCmd(cmd byte) error {
c.conn.SetWriteDeadline(time.Now().Add(c.timeout))
if _, err := c.conn.Write([]byte{cmd}); err != nil {
return errs.IO.WithErr(err)
}
return nil
}
func (c *client) dial() error {
c.conn = nil
conn, err := httpconn.Dial(c.endpoint)
func (c *client) dialConnect(endpoint string) error {
conn, err := httpconn.Dial(endpoint)
if err != nil {
return err
}
conn.SetWriteDeadline(time.Now().Add(c.timeout))
if _, err := conn.Write(c.psk); err != nil {
conn.Close()
if _, err := conn.Write(c.pskBytes[:]); err != nil {
return errs.IO.WithErr(err)
}
c.lock.Lock()
defer c.lock.Unlock()
c.conn = conn
return nil
}
func (c *client) withConn(cmd byte, fn func(net.Conn) error) error {
conn, err := c.getConn(cmd)
if err != nil {
return err
}
if err := fn(conn); err != nil {
conn.Close()
return err
}
return nil
}
func (c *client) getConn(cmd byte) (net.Conn, error) {
c.lock.Lock()
defer c.lock.Unlock()
if c.closed {
return nil, errs.IO.WithErr(io.EOF)
}
dialed := false
if c.conn == nil {
if err := c.dial(); err != nil {
return nil, err
}
dialed = true
}
if err := c.writeCmd(cmd); err != nil {
if dialed {
c.conn = nil
return nil, err
}
if err := c.dial(); err != nil {
return nil, err
}
if err := c.writeCmd(cmd); err != nil {
return nil, err
}
}
return c.conn, nil
}
func (c *client) recvJSON(
item any,
conn net.Conn,
timeout time.Duration,
) error {
if cap(c.buf) < 2 {
c.buf = make([]byte, 0, 1024)
}
buf := c.buf[:2]
conn.SetReadDeadline(time.Now().Add(timeout))
if _, err := io.ReadFull(conn, buf); err != nil {
return errs.IO.WithErr(err)
}
size := binary.LittleEndian.Uint16(buf)
if cap(buf) < int(size) {
buf = make([]byte, size)
c.buf = buf
}
buf = buf[:size]
if _, err := io.ReadFull(conn, buf); err != nil {
return errs.IO.WithErr(err)
}
if err := json.Unmarshal(buf, item); err != nil {
return errs.Unexpected.WithErr(err)
}
return nil
}

54
lib/rep/http-handler-util.go Normal file
View File

@ -0,0 +1,54 @@
package rep
import (
"crypto/subtle"
"io"
"log"
"net"
"net/http"
"time"
"git.crumpington.com/public/jldb/lib/httpconn"
)
func (rep *Replicator) handlerLogf(pattern string, args ...any) {
log.Printf("[HTTP-HANDLER] "+pattern, args...)
}
// checkBasicAuth tests whether the caller has provided the appropriate basic auth
// header. The caller should provide the PSK as the basic-auth password.
func (rep *Replicator) checkBasicAuth(w http.ResponseWriter, r *http.Request) bool {
_, pwd, _ := r.BasicAuth()
if subtle.ConstantTimeCompare([]byte(pwd), rep.pskBytes[:]) != 1 {
rep.handlerLogf("PSK mismatch.")
http.Error(w, "not authorized", http.StatusUnauthorized)
return false
}
return true
}
// acceptConnect accepts a CONNECT request and checks the PSK.
func (rep *Replicator) acceptConnect(w http.ResponseWriter, r *http.Request) net.Conn {
conn, err := httpconn.Accept(w, r)
if err != nil {
rep.handlerLogf("Failed to accept connection: %s", err)
return nil
}
psk := [64]byte{}
conn.SetReadDeadline(time.Now().Add(rep.conf.NetTimeout))
if _, err := io.ReadFull(conn, psk[:]); err != nil {
conn.Close()
rep.handlerLogf("Failed to read PSK: %v", err)
return nil
}
if subtle.ConstantTimeCompare(psk[:], rep.pskBytes[:]) != 1 {
conn.Close()
rep.handlerLogf("PSK mismatch.")
return nil
}
return conn
}

110
lib/rep/http-handler.go
View File

@ -1,79 +1,69 @@
package rep
import (
"crypto/subtle"
"git.crumpington.com/public/jldb/lib/httpconn"
"log"
"encoding/json"
"net/http"
"time"
"path"
)
const (
cmdGetInfo = 10
cmdSendState = 20
cmdStreamWAL = 30
pathGetInfo = "get-info"
pathSendState = "send-state"
pathStreamWAL = "stream-wal"
)
// ---------------------------------------------------------------------------
func (rep *Replicator) Handle(w http.ResponseWriter, r *http.Request) {
logf := func(pattern string, args ...any) {
log.Printf("[HTTP-HANDLER] "+pattern, args...)
// We'll handle two types of requests: HTTP GET requests for JSON, or
// streaming requets for state or wall.
base := path.Base(r.URL.Path)
switch base {
case pathGetInfo:
rep.handleGetInfo(w, r)
case pathSendState:
rep.handleSendState(w, r)
case pathStreamWAL:
rep.handleStreamWAL(w, r)
default:
http.Error(w, "not found", http.StatusNotFound)
}
}
func (rep *Replicator) handleGetInfo(w http.ResponseWriter, r *http.Request) {
if !rep.checkBasicAuth(w, r) {
return
}
conn, err := httpconn.Accept(w, r)
if err != nil {
logf("Failed to accept connection: %s", err)
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(rep.Info()); err != nil {
rep.handlerLogf("Failed to send info: %s", err)
}
}
func (rep *Replicator) handleSendState(w http.ResponseWriter, r *http.Request) {
conn := rep.acceptConnect(w, r)
if conn == nil {
return
}
defer conn.Close()
psk := make([]byte, 256)
conn.SetReadDeadline(time.Now().Add(rep.conf.NetTimeout))
if _, err := conn.Read(psk); err != nil {
logf("Failed to read PSK: %v", err)
return
}
expected := rep.pskBytes
if subtle.ConstantTimeCompare(expected, psk) != 1 {
logf("PSK mismatch.")
return
}
cmd := make([]byte, 1)
for {
conn.SetReadDeadline(time.Now().Add(rep.conf.NetTimeout))
if _, err := conn.Read(cmd); err != nil {
logf("Read failed: %v", err)
return
}
switch cmd[0] {
case cmdGetInfo:
if err := sendJSON(rep.Info(), conn, rep.conf.NetTimeout); err != nil {
logf("Failed to send info: %s", err)
return
}
case cmdSendState:
if err := rep.sendState(conn); err != nil {
if !rep.stopped() {
logf("Failed to send state: %s", err)
}
return
}
case cmdStreamWAL:
err := rep.wal.Send(conn, rep.conf.NetTimeout)
if !rep.stopped() {
logf("Failed when sending WAL: %s", err)
}
return
if err := rep.sendState(conn); err != nil {
if !rep.stopped() {
rep.handlerLogf("Failed to send state: %s", err)
}
}
}
func (rep *Replicator) handleStreamWAL(w http.ResponseWriter, r *http.Request) {
conn := rep.acceptConnect(w, r)
if conn == nil {
return
}
defer conn.Close()
err := rep.wal.Send(conn, rep.conf.NetTimeout)
if !rep.stopped() {
rep.handlerLogf("Failed when streaming WAL: %s", err)
}
}

17
lib/rep/paths.go Normal file
View File

@ -0,0 +1,17 @@
package rep
import (
"path/filepath"
)
func lockFilePath(rootDir string) string {
return filepath.Join(rootDir, "lock")
}
func walRootDir(rootDir string) string {
return filepath.Join(rootDir, "wal")
}
func stateFilePath(rootDir string) string {
return filepath.Join(rootDir, "state")
}

21
lib/rep/pools.go
View File

@ -1,21 +0,0 @@
package rep
import (
"bytes"
"sync"
)
var bufPool = sync.Pool{
New: func() any {
return &bytes.Buffer{}
},
}
func bufPoolGet() *bytes.Buffer {
return bufPool.Get().(*bytes.Buffer)
}
func bufPoolPut(b *bytes.Buffer) {
b.Reset()
bufPool.Put(b)
}

16
lib/rep/psk.go Normal file
View File

@ -0,0 +1,16 @@
package rep
import (
"crypto/sha256"
"encoding/hex"
)
func pskToBytes(in string) [64]byte {
b := sha256.Sum256([]byte(in))
dst := [64]byte{}
i := hex.Encode(dst[:], b[:])
if i != 64 {
panic(i)
}
return dst
}

3
lib/rep/rep-sendrecv.go
View File

@ -2,9 +2,10 @@ package rep
import (
"io"
"git.crumpington.com/public/jldb/lib/errs"
"net"
"time"
"git.crumpington.com/public/jldb/lib/errs"
)
func (rep *Replicator) sendState(conn net.Conn) error {

9
lib/rep/replicator-open.go
View File

@ -1,12 +1,13 @@
package rep
import (
"os"
"time"
"git.crumpington.com/public/jldb/lib/atomicheader"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/flock"
"git.crumpington.com/public/jldb/lib/wal"
"os"
"time"
)
func (rep *Replicator) loadConfigDefaults() {
@ -26,9 +27,7 @@ func (rep *Replicator) loadConfigDefaults() {
}
rep.conf = conf
rep.pskBytes = make([]byte, 256)
copy(rep.pskBytes, []byte(conf.ReplicationPSK))
rep.pskBytes = pskToBytes(conf.ReplicationPSK)
}
func (rep *Replicator) initDirectories() error {

7
lib/rep/replicator-walrecvr.go
View File

@ -30,9 +30,8 @@ func (rep *Replicator) runWALRecvrOnce() {
log.Printf("[WAL-RECVR] "+pattern, args...)
}
if err := rep.client.StreamWAL(rep.wal); err != nil {
if !rep.stopped() {
logf("Recv failed: %v", err)
}
err := rep.client.StreamWAL(rep.wal)
if !rep.stopped() {
logf("Recv failed: %v", err)
}
}

16
lib/rep/replicator.go
View File

@ -2,14 +2,15 @@ package rep
import (
"io"
"git.crumpington.com/public/jldb/lib/atomicheader"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/wal"
"net"
"os"
"sync"
"sync/atomic"
"time"
"git.crumpington.com/public/jldb/lib/atomicheader"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/wal"
)
type Config struct {
@ -59,8 +60,9 @@ type Replicator struct {
conf Config
lockFile *os.File
pskBytes []byte
wal *wal.WAL
pskBytes [64]byte // 64 ascii characters. See pskToBytes.
wal *wal.WAL
appendNotify chan struct{}
@ -161,10 +163,6 @@ func (rep *Replicator) Primary() bool {
return rep.conf.Primary
}
// TODO: Probably remove this.
// The caller may call Ack after Apply to acknowledge that the change has also
// been applied to the caller's application. Alternatively, the caller may use
// follow to apply changes to their application state.
func (rep *Replicator) ack(seqNum, timestampMS int64) error {
state := rep.getState()
state.SeqNum = seqNum

6
lib/rep/testapp_test.go
View File

@ -5,12 +5,14 @@ import (
"encoding/binary"
"encoding/json"
"io"
"git.crumpington.com/public/jldb/lib/wal"
"math/rand"
"net"
"sync"
"testing"
"time"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/wal"
)
// ----------------------------------------------------------------------------
@ -82,7 +84,7 @@ func newApp(t *testing.T, id int64, conf Config) *TestApp {
Apply: a.apply,
}, conf)
if err != nil {
t.Fatal(err)
t.Fatal(errs.FmtDetails(err))
}
return a

3
lib/wal/corrupt_test.go
View File

@ -2,8 +2,9 @@ package wal
import (
"io"
"git.crumpington.com/public/jldb/lib/errs"
"testing"
"git.crumpington.com/public/jldb/lib/errs"
)
func TestCorruptWAL(t *testing.T) {

3
lib/wal/generic_test.go
View File

@ -5,13 +5,14 @@ import (
"encoding/binary"
"errors"
"io"
"git.crumpington.com/public/jldb/lib/errs"
"math/rand"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"git.crumpington.com/public/jldb/lib/errs"
)
type waLog interface {

1
lib/wal/io.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"hash/crc32"
"io"
"git.crumpington.com/public/jldb/lib/errs"
)

1
lib/wal/record.go
View File

@ -4,6 +4,7 @@ import (
"encoding/binary"
"hash/crc32"
"io"
"git.crumpington.com/public/jldb/lib/errs"
)

5
lib/wal/record_test.go
View File

@ -3,10 +3,11 @@ package wal
import (
"bytes"
"io"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/testutil"
"math/rand"
"testing"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/testutil"
)
func NewRecordForTesting() Record {

5
lib/wal/segment-iterator.go
View File

@ -1,10 +1,11 @@
package wal
import (
"git.crumpington.com/public/jldb/lib/atomicheader"
"git.crumpington.com/public/jldb/lib/errs"
"os"
"time"
"git.crumpington.com/public/jldb/lib/atomicheader"
"git.crumpington.com/public/jldb/lib/errs"
)
type segmentIterator struct {

5
lib/wal/segment.go
View File

@ -3,11 +3,12 @@ package wal
import (
"bufio"
"io"
"git.crumpington.com/public/jldb/lib/atomicheader"
"git.crumpington.com/public/jldb/lib/errs"
"os"
"sync"
"time"
"git.crumpington.com/public/jldb/lib/atomicheader"
"git.crumpington.com/public/jldb/lib/errs"
)
type segment struct {

5
lib/wal/segment_test.go
View File

@ -4,11 +4,12 @@ import (
"bytes"
crand "crypto/rand"
"io"
"git.crumpington.com/public/jldb/lib/atomicheader"
"git.crumpington.com/public/jldb/lib/errs"
"path/filepath"
"testing"
"time"
"git.crumpington.com/public/jldb/lib/atomicheader"
"git.crumpington.com/public/jldb/lib/errs"
)
func newSegmentForTesting(t *testing.T) *segment {

3
lib/wal/wal-iterator.go
View File

@ -1,8 +1,9 @@
package wal
import (
"git.crumpington.com/public/jldb/lib/errs"
"time"
"git.crumpington.com/public/jldb/lib/errs"
)
type walIterator struct {

3
lib/wal/wal-recv.go
View File

@ -3,9 +3,10 @@ package wal
import (
"encoding/binary"
"io"
"git.crumpington.com/public/jldb/lib/errs"
"net"
"time"
"git.crumpington.com/public/jldb/lib/errs"
)
func (wal *WAL) Recv(conn net.Conn, timeout time.Duration) error {

3
lib/wal/wal-send.go
View File

@ -2,9 +2,10 @@ package wal
import (
"encoding/binary"
"git.crumpington.com/public/jldb/lib/errs"
"net"
"time"
"git.crumpington.com/public/jldb/lib/errs"
)
const (

5
lib/wal/wal-sendrecv_test.go
View File

@ -1,8 +1,6 @@
package wal
import (
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/testutil"
"log"
"math/rand"
"reflect"
@ -11,6 +9,9 @@ import (
"sync/atomic"
"testing"
"time"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/testutil"
)
func TestSendRecvHarness(t *testing.T) {

5
lib/wal/wal.go
View File

@ -2,13 +2,14 @@ package wal
import (
"io"
"git.crumpington.com/public/jldb/lib/atomicheader"
"git.crumpington.com/public/jldb/lib/errs"
"os"
"path/filepath"
"strconv"
"sync"
"time"
"git.crumpington.com/public/jldb/lib/atomicheader"
"git.crumpington.com/public/jldb/lib/errs"
)
type Config struct {

1
mdb/change/binary.go
View File

@ -3,6 +3,7 @@ package change
import (
"encoding/binary"
"io"
"git.crumpington.com/public/jldb/lib/errs"
)

1
mdb/change/change.go
View File

@ -2,6 +2,7 @@ package change
import (
"io"
"git.crumpington.com/public/jldb/lib/errs"
)

62
mdb/collection.go
View File

@ -5,9 +5,10 @@ import (
"encoding/json"
"errors"
"hash/crc64"
"git.crumpington.com/public/jldb/lib/errs"
"unsafe"
"git.crumpington.com/public/jldb/lib/errs"
"github.com/google/btree"
)
@ -20,10 +21,10 @@ type Collection[T any] struct {
sanitize func(*T)
validate func(*T) error
indices []Index[T]
uniqueIndices []Index[T]
indices []*Index[T]
uniqueIndices []*Index[T]
ByID Index[T]
ByID *Index[T]
buf *bytes.Buffer
}
@ -64,8 +65,8 @@ func NewCollection[T any](db *Database, name string, conf *CollectionConfig[T])
copy: conf.Copy,
sanitize: conf.Sanitize,
validate: conf.Validate,
indices: []Index[T]{},
uniqueIndices: []Index[T]{},
indices: []*Index[T]{},
uniqueIndices: []*Index[T]{},
buf: &bytes.Buffer{},
}
@ -91,7 +92,7 @@ func NewCollection[T any](db *Database, name string, conf *CollectionConfig[T])
return c
}
func (c Collection[T]) Name() string {
func (c *Collection[T]) Name() string {
return c.name
}
@ -107,13 +108,13 @@ type indexConfig[T any] struct {
Include func(item *T) bool
}
func (c Collection[T]) Get(tx *Snapshot, id uint64) (*T, bool) {
func (c *Collection[T]) Get(tx *Snapshot, id uint64) (*T, bool) {
x := new(T)
c.setID(x, id)
return c.ByID.Get(tx, x)
}
func (c Collection[T]) List(tx *Snapshot, ids []uint64, out []*T) []*T {
func (c *Collection[T]) List(tx *Snapshot, ids []uint64, out []*T) []*T {
if len(ids) == 0 {
return out[:0]
}
@ -134,8 +135,7 @@ func (c Collection[T]) List(tx *Snapshot, ids []uint64, out []*T) []*T {
}
// AddIndex: Add an index to the collection.
func (c *Collection[T]) addIndex(conf indexConfig[T]) Index[T] {
func (c *Collection[T]) addIndex(conf indexConfig[T]) *Index[T] {
var less func(*T, *T) bool
if conf.Unique {
@ -159,7 +159,7 @@ func (c *Collection[T]) addIndex(conf indexConfig[T]) Index[T] {
BTree: btree.NewG(256, less),
}
index := Index[T]{
index := &Index[T]{
collectionID: c.collectionID,
name: conf.Name,
indexID: c.getState(c.db.Snapshot()).addIndex(indexState),
@ -175,7 +175,7 @@ func (c *Collection[T]) addIndex(conf indexConfig[T]) Index[T] {
return index
}
func (c Collection[T]) Insert(tx *Snapshot, userItem *T) error {
func (c *Collection[T]) Insert(tx *Snapshot, userItem *T) error {
if err := c.ensureMutable(tx); err != nil {
return err
}
@ -189,7 +189,7 @@ func (c Collection[T]) Insert(tx *Snapshot, userItem *T) error {
for i := range c.uniqueIndices {
if c.uniqueIndices[i].insertConflict(tx, item) {
return ErrDuplicate.WithCollection(c.name).WithIndex(c.uniqueIndices[i].name)
return errs.Duplicate.WithCollection(c.name).WithIndex(c.uniqueIndices[i].name)
}
}
@ -202,7 +202,7 @@ func (c Collection[T]) Insert(tx *Snapshot, userItem *T) error {
return nil
}
func (c Collection[T]) Update(tx *Snapshot, userItem *T) error {
func (c *Collection[T]) Update(tx *Snapshot, userItem *T) error {
if err := c.ensureMutable(tx); err != nil {
return err
}
@ -216,12 +216,12 @@ func (c Collection[T]) Update(tx *Snapshot, userItem *T) error {
old, ok := c.ByID.get(tx, item)
if !ok {
return ErrNotFound
return errs.NotFound
}
for i := range c.uniqueIndices {
if c.uniqueIndices[i].updateConflict(tx, item) {
return ErrDuplicate.WithCollection(c.name).WithIndex(c.uniqueIndices[i].name)
return errs.Duplicate.WithCollection(c.name).WithIndex(c.uniqueIndices[i].name)
}
}
@ -234,18 +234,18 @@ func (c Collection[T]) Update(tx *Snapshot, userItem *T) error {
return nil
}
func (c Collection[T]) Upsert(tx *Snapshot, item *T) error {
func (c *Collection[T]) Upsert(tx *Snapshot, item *T) error {
err := c.Insert(tx, item)
if err == nil {
return nil
}
if errors.Is(err, ErrDuplicate) {
if errors.Is(err, errs.Duplicate) {
return c.Update(tx, item)
}
return err
}
func (c Collection[T]) Delete(tx *Snapshot, itemID uint64) error {
func (c *Collection[T]) Delete(tx *Snapshot, itemID uint64) error {
if err := c.ensureMutable(tx); err != nil {
return err
}
@ -253,15 +253,15 @@ func (c Collection[T]) Delete(tx *Snapshot, itemID uint64) error {
return c.deleteItem(tx, itemID)
}
func (c Collection[T]) getByID(tx *Snapshot, itemID uint64) (*T, bool) {
func (c *Collection[T]) getByID(tx *Snapshot, itemID uint64) (*T, bool) {
x := new(T)
c.setID(x, itemID)
return c.ByID.get(tx, x)
}
func (c Collection[T]) ensureMutable(tx *Snapshot) error {
func (c *Collection[T]) ensureMutable(tx *Snapshot) error {
if !tx.writable() {
return ErrReadOnly
return errs.ReadOnly
}
state := c.getState(tx)
@ -273,7 +273,7 @@ func (c Collection[T]) ensureMutable(tx *Snapshot) error {
}
// For initial data loading.
func (c Collection[T]) insertItem(tx *Snapshot, itemID uint64, data []byte) error {
func (c *Collection[T]) insertItem(tx *Snapshot, itemID uint64, data []byte) error {
item := new(T)
if err := json.Unmarshal(data, item); err != nil {
return errs.Encoding.WithErr(err).WithCollection(c.name)
@ -282,7 +282,7 @@ func (c Collection[T]) insertItem(tx *Snapshot, itemID uint64, data []byte) erro
// Check for insert conflict.
for _, index := range c.uniqueIndices {
if index.insertConflict(tx, item) {
return ErrDuplicate
return errs.Duplicate
}
}
@ -294,10 +294,10 @@ func (c Collection[T]) insertItem(tx *Snapshot, itemID uint64, data []byte) erro
return nil
}
func (c Collection[T]) deleteItem(tx *Snapshot, itemID uint64) error {
func (c *Collection[T]) deleteItem(tx *Snapshot, itemID uint64) error {
item, ok := c.getByID(tx, itemID)
if !ok {
return ErrNotFound
return errs.NotFound
}
tx.delete(c.collectionID, itemID)
@ -311,7 +311,7 @@ func (c Collection[T]) deleteItem(tx *Snapshot, itemID uint64) error {
// upsertItem inserts or updates the item with itemID and the given serialized
// form. It's called by
func (c Collection[T]) upsertItem(tx *Snapshot, itemID uint64, data []byte) error {
func (c *Collection[T]) upsertItem(tx *Snapshot, itemID uint64, data []byte) error {
item, ok := c.getByID(tx, itemID)
if ok {
tx.delete(c.collectionID, itemID)
@ -334,14 +334,14 @@ func (c Collection[T]) upsertItem(tx *Snapshot, itemID uint64, data []byte) erro
return nil
}
func (c Collection[T]) getID(t *T) uint64 {
func (c *Collection[T]) getID(t *T) uint64 {
return *((*uint64)(unsafe.Pointer(t)))
}
func (c Collection[T]) setID(t *T, id uint64) {
func (c *Collection[T]) setID(t *T, id uint64) {
*((*uint64)(unsafe.Pointer(t))) = id
}
func (c Collection[T]) getState(tx *Snapshot) *collectionState[T] {
func (c *Collection[T]) getState(tx *Snapshot) *collectionState[T] {
return tx.collections[c.collectionID].(*collectionState[T])
}

3
mdb/crashconsistency_test.go
View File

@ -1,12 +1,13 @@
package mdb
import (
"git.crumpington.com/public/jldb/lib/errs"
"log"
"os"
"os/exec"
"testing"
"time"
"git.crumpington.com/public/jldb/lib/errs"
)
func TestCrashConsistency(t *testing.T) {

67
mdb/db-primary.go
View File

@ -1,67 +0,0 @@
package mdb
/*
func (db *Database) openPrimary() (err error) {
wal, err := cwal.Open(db.walRootDir, cwal.Config{
SegMinCount: db.conf.WALSegMinCount,
SegMaxAgeSec: db.conf.WALSegMaxAgeSec,
})
pFile, err := pfile.Open(db.pageFilePath,
pFile, err := openPageFileAndReplayWAL(db.rootDir)
if err != nil {
return err
}
defer pFile.Close()
pfHeader, err := pFile.ReadHeader()
if err != nil {
return err
}
tx := db.Snapshot()
tx.seqNum = pfHeader.SeqNum
tx.updatedAt = pfHeader.UpdatedAt
pIndex, err := pagefile.NewIndex(pFile)
if err != nil {
return err
}
err = pFile.IterateAllocated(pIndex, func(cID, iID uint64, data []byte) error {
return db.loadItem(tx, cID, iID, data)
})
if err != nil {
return err
}
w, err := cwal.OpenWriter(db.walRootDir, &cwal.WriterConfig{
SegMinCount: db.conf.WALSegMinCount,
SegMaxAgeSec: db.conf.WALSegMaxAgeSec,
})
if err != nil {
return err
}
db.done.Add(1)
go txAggregator{
Stop: db.stop,
Done: db.done,
ModChan: db.modChan,
W: w,
Index: pIndex,
Snapshot: db.snapshot,
}.Run()
db.done.Add(1)
go (&fileWriter{
Stop: db.stop,
Done: db.done,
PageFilePath: db.pageFilePath,
WALRootDir: db.walRootDir,
}).Run()
return nil
}
*/

7
mdb/db-rep.go
View File

@ -1,13 +1,14 @@
package mdb
import (
"log"
"net"
"os"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/wal"
"git.crumpington.com/public/jldb/mdb/change"
"git.crumpington.com/public/jldb/mdb/pfile"
"log"
"net"
"os"
)
func (db *Database) repSendState(conn net.Conn) error {

129
mdb/db-secondary.go
View File

@ -1,129 +0,0 @@
package mdb
/*
func (db *Database) openSecondary() (err error) {
if db.shouldLoadFromPrimary() {
if err := db.loadFromPrimary(); err != nil {
return err
}
}
log.Printf("Opening page-file...")
pFile, err := openPageFileAndReplayWAL(db.rootDir)
if err != nil {
return err
}
defer pFile.Close()
pfHeader, err := pFile.ReadHeader()
if err != nil {
return err
}
log.Printf("Building page-file index...")
pIndex, err := pagefile.NewIndex(pFile)
if err != nil {
return err
}
tx := db.Snapshot()
tx.seqNum = pfHeader.SeqNum
tx.updatedAt = pfHeader.UpdatedAt
log.Printf("Loading data into memory...")
err = pFile.IterateAllocated(pIndex, func(cID, iID uint64, data []byte) error {
return db.loadItem(tx, cID, iID, data)
})
if err != nil {
return err
}
log.Printf("Creating writer...")
w, err := cswal.OpenWriter(db.walRootDir, &cswal.WriterConfig{
SegMinCount: db.conf.WALSegMinCount,
SegMaxAgeSec: db.conf.WALSegMaxAgeSec,
})
if err != nil {
return err
}
db.done.Add(1)
go (&walFollower{
Stop: db.stop,
Done: db.done,
W: w,
Client: NewClient(db.conf.PrimaryURL, db.conf.ReplicationPSK, db.conf.NetTimeout),
}).Run()
db.done.Add(1)
go (&follower{
Stop: db.stop,
Done: db.done,
WALRootDir: db.walRootDir,
SeqNum: pfHeader.SeqNum,
ApplyChanges: db.applyChanges,
}).Run()
db.done.Add(1)
go (&fileWriter{
Stop: db.stop,
Done: db.done,
PageFilePath: db.pageFilePath,
WALRootDir: db.walRootDir,
}).Run()
return nil
}
func (db *Database) shouldLoadFromPrimary() bool {
if _, err := os.Stat(db.walRootDir); os.IsNotExist(err) {
log.Printf("WAL doesn't exist.")
return true
}
if _, err := os.Stat(db.pageFilePath); os.IsNotExist(err) {
log.Printf("Page-file doesn't exist.")
return true
}
return false
}
func (db *Database) loadFromPrimary() error {
client := NewClient(db.conf.PrimaryURL, db.conf.ReplicationPSK, db.conf.NetTimeout)
defer client.Disconnect()
log.Printf("Loading data from primary...")
if err := os.RemoveAll(db.pageFilePath); err != nil {
log.Printf("Failed to remove page-file: %s", err)
return errs.IO.WithErr(err) // Caller can retry.
}
if err := os.RemoveAll(db.walRootDir); err != nil {
log.Printf("Failed to remove WAL: %s", err)
return errs.IO.WithErr(err) // Caller can retry.
}
err := client.DownloadPageFile(db.pageFilePath+".tmp", db.pageFilePath)
if err != nil {
log.Printf("Failed to get page-file from primary: %s", err)
return err // Caller can retry.
}
pfHeader, err := pagefile.ReadHeader(db.pageFilePath)
if err != nil {
log.Printf("Failed to read page-file sequence number: %s", err)
return err // Caller can retry.
}
if err = cswal.CreateEx(db.walRootDir, pfHeader.SeqNum+1); err != nil {
log.Printf("Failed to initialize WAL: %s", err)
return err // Caller can retry.
}
return nil
}
*/

34
mdb/db-testcases_test.go
View File

@ -6,6 +6,8 @@ import (
"reflect"
"strings"
"testing"
"git.crumpington.com/public/jldb/lib/errs"
)
type DBTestCase struct {
@ -54,7 +56,7 @@ var testDBTestCases = []DBTestCase{{
Update: func(t *testing.T, db TestDB, tx *Snapshot) error {
user, ok := db.Users.ByID.Get(tx, &User{ID: 1})
if !ok {
return ErrNotFound
return errs.NotFound
}
user.Name = "Bob"
user.Email = "b@c.com"
@ -111,7 +113,7 @@ var testDBTestCases = []DBTestCase{{
return db.Users.Insert(tx, user2)
},
ExpectedUpdateError: ErrDuplicate,
ExpectedUpdateError: errs.Duplicate,
State: DBState{},
}},
@ -131,7 +133,7 @@ var testDBTestCases = []DBTestCase{{
return db.Users.Insert(tx, user2)
},
ExpectedUpdateError: ErrDuplicate,
ExpectedUpdateError: errs.Duplicate,
State: DBState{},
}},
@ -162,7 +164,7 @@ var testDBTestCases = []DBTestCase{{
return db.Users.Insert(tx, user)
},
ExpectedUpdateError: ErrDuplicate,
ExpectedUpdateError: errs.Duplicate,
State: DBState{
UsersByID: []User{{ID: 1, Name: "Alice", Email: "a@b.com"}},
@ -197,7 +199,7 @@ var testDBTestCases = []DBTestCase{{
return db.Users.Insert(tx, user)
},
ExpectedUpdateError: ErrDuplicate,
ExpectedUpdateError: errs.Duplicate,
State: DBState{
UsersByID: []User{{ID: 1, Name: "Alice", Email: "a@b.com"}},
@ -218,7 +220,7 @@ var testDBTestCases = []DBTestCase{{
return db.Users.Insert(db.Snapshot(), user)
},
ExpectedUpdateError: ErrReadOnly,
ExpectedUpdateError: errs.ReadOnly,
}},
}, {
@ -290,7 +292,7 @@ var testDBTestCases = []DBTestCase{{
return db.Users.Update(tx, user)
},
ExpectedUpdateError: ErrNotFound,
ExpectedUpdateError: errs.NotFound,
State: DBState{
UsersByID: []User{{ID: 5, Name: "Alice", Email: "a@b.com"}},
@ -323,14 +325,14 @@ var testDBTestCases = []DBTestCase{{
Update: func(t *testing.T, db TestDB, tx *Snapshot) error {
user, ok := db.Users.ByID.Get(tx, &User{ID: 1})
if !ok {
return ErrNotFound
return errs.NotFound
}
user.Name = "Bob"
user.Email = "b@c.com"
return db.Users.Update(db.Snapshot(), user)
},
ExpectedUpdateError: ErrReadOnly,
ExpectedUpdateError: errs.ReadOnly,
State: DBState{
UsersByID: []User{{ID: 1, Name: "Alice", Email: "a@b.com"}},
@ -451,7 +453,7 @@ var testDBTestCases = []DBTestCase{{
return db.Users.Update(tx, user2)
},
ExpectedUpdateError: ErrDuplicate,
ExpectedUpdateError: errs.Duplicate,
State: DBState{},
}},
@ -493,14 +495,14 @@ var testDBTestCases = []DBTestCase{{
Update: func(t *testing.T, db TestDB, tx *Snapshot) error {
u, ok := db.Users.ByID.Get(tx, &User{ID: 2})
if !ok {
return ErrNotFound
return errs.NotFound
}
u.Email = "a@b.com"
return db.Users.Update(tx, u)
},
ExpectedUpdateError: ErrDuplicate,
ExpectedUpdateError: errs.Duplicate,
State: DBState{
UsersByID: []User{
@ -542,7 +544,7 @@ var testDBTestCases = []DBTestCase{{
return db.Users.Delete(db.Snapshot(), 1)
},
ExpectedUpdateError: ErrReadOnly,
ExpectedUpdateError: errs.ReadOnly,
State: DBState{
UsersByID: []User{{ID: 1, Name: "Alice", Email: "a@b.com"}},
@ -575,7 +577,7 @@ var testDBTestCases = []DBTestCase{{
return db.Users.Delete(tx, 2)
},
ExpectedUpdateError: ErrNotFound,
ExpectedUpdateError: errs.NotFound,
State: DBState{
UsersByID: []User{{ID: 1, Name: "Alice", Email: "a@b.com"}},
@ -609,7 +611,7 @@ var testDBTestCases = []DBTestCase{{
u, ok := db.Users.ByID.Get(tx, &User{ID: 1})
if !ok {
return ErrNotFound
return errs.NotFound
}
if !reflect.DeepEqual(u, expected) {
return errors.New("Not equal (id)")
@ -617,7 +619,7 @@ var testDBTestCases = []DBTestCase{{
u, ok = db.Users.ByEmail.Get(tx, &User{Email: "a@b.com"})
if !ok {
return ErrNotFound
return errs.NotFound
}
if !reflect.DeepEqual(u, expected) {
return errors.New("Not equal (email)")

2
mdb/db-testrunner_test.go
View File

@ -134,7 +134,7 @@ func checkSlicesEqual[T any](t *testing.T, name string, actual, expected []T) {
}
}
func checkMinMaxEqual[T any](t *testing.T, name string, tx *Snapshot, index Index[T], expected []T) {
func checkMinMaxEqual[T any](t *testing.T, name string, tx *Snapshot, index *Index[T], expected []T) {
if len(expected) == 0 {
if min, ok := index.Min(tx); ok {
t.Fatal(min)

1
mdb/db-txaggregator.go
View File

@ -2,6 +2,7 @@ package mdb
import (
"bytes"
"git.crumpington.com/public/jldb/mdb/change"
)

2
mdb/db-userdata_test.go
View File

@ -14,7 +14,7 @@ type UserDataItem struct {
type UserData struct {
*Collection[UserDataItem]
ByName Index[UserDataItem] // Unique index on (Token).
ByName *Index[UserDataItem] // Unique index on (Token).
}
func NewUserDataCollection(db *Database) UserData {

6
mdb/db-users_test.go
View File

@ -12,9 +12,9 @@ type User struct {
type Users struct {
*Collection[User]
ByEmail Index[User] // Unique index on (Email).
ByName Index[User] // Index on (Name).
ByBlocked Index[User] // Partial index on (Blocked,Email).
ByEmail *Index[User] // Unique index on (Email).
ByName *Index[User] // Index on (Name).
ByBlocked *Index[User] // Partial index on (Blocked,Email).
}
func NewUserCollection(db *Database) Users {

9
mdb/db.go
View File

@ -2,15 +2,16 @@ package mdb
import (
"fmt"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/rep"
"git.crumpington.com/public/jldb/mdb/change"
"git.crumpington.com/public/jldb/mdb/pfile"
"net/http"
"os"
"sync"
"sync/atomic"
"time"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/lib/rep"
"git.crumpington.com/public/jldb/mdb/change"
"git.crumpington.com/public/jldb/mdb/pfile"
)
type Config struct {

11
mdb/errors.go
View File

@ -1,11 +0,0 @@
package mdb
import (
"git.crumpington.com/public/jldb/lib/errs"
)
var (
ErrNotFound = errs.NotFound
ErrReadOnly = errs.ReadOnly
ErrDuplicate = errs.Duplicate
)

46
mdb/index.go
View File

@ -10,7 +10,7 @@ func NewIndex[T any](
c *Collection[T],
name string,
compare func(lhs, rhs *T) int,
) Index[T] {
) *Index[T] {
return c.addIndex(indexConfig[T]{
Name: name,
Unique: false,
@ -24,7 +24,7 @@ func NewPartialIndex[T any](
name string,
compare func(lhs, rhs *T) int,
include func(*T) bool,
) Index[T] {
) *Index[T] {
return c.addIndex(indexConfig[T]{
Name: name,
Unique: false,
@ -37,7 +37,7 @@ func NewUniqueIndex[T any](
c *Collection[T],
name string,
compare func(lhs, rhs *T) int,
) Index[T] {
) *Index[T] {
return c.addIndex(indexConfig[T]{
Name: name,
Unique: true,
@ -51,7 +51,7 @@ func NewUniquePartialIndex[T any](
name string,
compare func(lhs, rhs *T) int,
include func(*T) bool,
) Index[T] {
) *Index[T] {
return c.addIndex(indexConfig[T]{
Name: name,
Unique: true,
@ -70,7 +70,7 @@ type Index[T any] struct {
copy func(*T) *T
}
func (i Index[T]) Get(tx *Snapshot, in *T) (item *T, ok bool) {
func (i *Index[T]) Get(tx *Snapshot, in *T) (item *T, ok bool) {
tPtr, ok := i.get(tx, in)
if !ok {
return item, false
@ -78,15 +78,15 @@ func (i Index[T]) Get(tx *Snapshot, in *T) (item *T, ok bool) {
return i.copy(tPtr), true
}
func (i Index[T]) get(tx *Snapshot, in *T) (*T, bool) {
func (i *Index[T]) get(tx *Snapshot, in *T) (*T, bool) {
return i.btree(tx).Get(in)
}
func (i Index[T]) Has(tx *Snapshot, in *T) bool {
func (i *Index[T]) Has(tx *Snapshot, in *T) bool {
return i.btree(tx).Has(in)
}
func (i Index[T]) Min(tx *Snapshot) (item *T, ok bool) {
func (i *Index[T]) Min(tx *Snapshot) (item *T, ok bool) {
tPtr, ok := i.btree(tx).Min()
if !ok {
return item, false
@ -94,7 +94,7 @@ func (i Index[T]) Min(tx *Snapshot) (item *T, ok bool) {
return i.copy(tPtr), true
}
func (i Index[T]) Max(tx *Snapshot) (item *T, ok bool) {
func (i *Index[T]) Max(tx *Snapshot) (item *T, ok bool) {
tPtr, ok := i.btree(tx).Max()
if !ok {
return item, false
@ -102,25 +102,25 @@ func (i Index[T]) Max(tx *Snapshot) (item *T, ok bool) {
return i.copy(tPtr), true
}
func (i Index[T]) Ascend(tx *Snapshot, each func(*T) bool) {
func (i *Index[T]) Ascend(tx *Snapshot, each func(*T) bool) {
i.btreeForIter(tx).Ascend(func(t *T) bool {
return each(i.copy(t))
})
}
func (i Index[T]) AscendAfter(tx *Snapshot, after *T, each func(*T) bool) {
func (i *Index[T]) AscendAfter(tx *Snapshot, after *T, each func(*T) bool) {
i.btreeForIter(tx).AscendGreaterOrEqual(after, func(t *T) bool {
return each(i.copy(t))
})
}
func (i Index[T]) Descend(tx *Snapshot, each func(*T) bool) {
func (i *Index[T]) Descend(tx *Snapshot, each func(*T) bool) {
i.btreeForIter(tx).Descend(func(t *T) bool {
return each(i.copy(t))
})
}
func (i Index[T]) DescendAfter(tx *Snapshot, after *T, each func(*T) bool) {
func (i *Index[T]) DescendAfter(tx *Snapshot, after *T, each func(*T) bool) {
i.btreeForIter(tx).DescendLessOrEqual(after, func(t *T) bool {
return each(i.copy(t))
})
@ -133,7 +133,7 @@ type ListArgs[T any] struct {
Limit int // Maximum number of items to return. 0 => All.
}
func (i Index[T]) List(tx *Snapshot, args ListArgs[T], out []*T) []*T {
func (i *Index[T]) List(tx *Snapshot, args ListArgs[T], out []*T) []*T {
if args.Limit < 0 {
return nil
}
@ -176,11 +176,11 @@ func (i Index[T]) List(tx *Snapshot, args ListArgs[T], out []*T) []*T {
// ----------------------------------------------------------------------------
func (i Index[T]) insertConflict(tx *Snapshot, item *T) bool {
func (i *Index[T]) insertConflict(tx *Snapshot, item *T) bool {
return i.btree(tx).Has(item)
}
func (i Index[T]) updateConflict(tx *Snapshot, item *T) bool {
func (i *Index[T]) updateConflict(tx *Snapshot, item *T) bool {
current, ok := i.btree(tx).Get(item)
return ok && i.getID(current) != i.getID(item)
}
@ -188,7 +188,7 @@ func (i Index[T]) updateConflict(tx *Snapshot, item *T) bool {
// This should only be called after insertConflict. Additionally, the caller
// should ensure that the index has been properly cloned for write before
// writing.
func (i Index[T]) insert(tx *Snapshot, item *T) {
func (i *Index[T]) insert(tx *Snapshot, item *T) {
if i.include != nil && !i.include(item) {
return
}
@ -196,7 +196,7 @@ func (i Index[T]) insert(tx *Snapshot, item *T) {
i.btree(tx).ReplaceOrInsert(item)
}
func (i Index[T]) update(tx *Snapshot, old, new *T) {
func (i *Index[T]) update(tx *Snapshot, old, new *T) {
bt := i.btree(tx)
bt.Delete(old)
@ -204,22 +204,22 @@ func (i Index[T]) update(tx *Snapshot, old, new *T) {
i.insert(tx, new)
}
func (i Index[T]) delete(tx *Snapshot, item *T) {
func (i *Index[T]) delete(tx *Snapshot, item *T) {
i.btree(tx).Delete(item)
}
// ----------------------------------------------------------------------------
func (i Index[T]) getState(tx *Snapshot) indexState[T] {
func (i *Index[T]) getState(tx *Snapshot) indexState[T] {
return tx.collections[i.collectionID].(*collectionState[T]).Indices[i.indexID]
}
// Get the current btree for get/has/update/delete, etc.
func (i Index[T]) btree(tx *Snapshot) *btree.BTreeG[*T] {
func (i *Index[T]) btree(tx *Snapshot) *btree.BTreeG[*T] {
return i.getState(tx).BTree
}
func (i Index[T]) btreeForIter(tx *Snapshot) *btree.BTreeG[*T] {
func (i *Index[T]) btreeForIter(tx *Snapshot) *btree.BTreeG[*T] {
cState := tx.collections[i.collectionID].(*collectionState[T])
bt := cState.Indices[i.indexID].BTree
@ -231,6 +231,6 @@ func (i Index[T]) btreeForIter(tx *Snapshot) *btree.BTreeG[*T] {
return bt
}
func (i Index[T]) getID(t *T) uint64 {
func (i *Index[T]) getID(t *T) uint64 {
return *((*uint64)(unsafe.Pointer(t)))
}

3
mdb/pfile/change_test.go
View File

@ -2,8 +2,9 @@ package pfile
import (
crand "crypto/rand"
"git.crumpington.com/public/jldb/mdb/change"
"math/rand"
"git.crumpington.com/public/jldb/mdb/change"
)
func randomChangeList() (changes []change.Change) {

5
mdb/pfile/main_test.go
View File

@ -3,10 +3,11 @@ package pfile
import (
"bytes"
crand "crypto/rand"
"git.crumpington.com/public/jldb/lib/wal"
"git.crumpington.com/public/jldb/mdb/change"
"path/filepath"
"testing"
"git.crumpington.com/public/jldb/lib/wal"
"git.crumpington.com/public/jldb/mdb/change"
)
func newForTesting(t *testing.T) (*File, *Index) {

5
mdb/pfile/page.go
View File

@ -2,8 +2,9 @@ package pfile
import (
"hash/crc32"
"git.crumpington.com/public/jldb/lib/errs"
"unsafe"
"git.crumpington.com/public/jldb/lib/errs"
)
// ----------------------------------------------------------------------------
@ -30,7 +31,7 @@ var emptyPage = func() dataPage {
type pageHeader struct {
CRC uint32 // IEEE CRC-32 checksum.
PageType uint32 // One of the PageType* constants.
CollectionID uint64 //
CollectionID uint64 //
ItemID uint64
DataSize uint64
NextPage uint64

3
mdb/pfile/page_test.go
View File

@ -3,9 +3,10 @@ package pfile
import (
"bytes"
crand "crypto/rand"
"git.crumpington.com/public/jldb/lib/errs"
"math/rand"
"testing"
"git.crumpington.com/public/jldb/lib/errs"
)
func randomPage(t *testing.T) dataPage {

5
mdb/pfile/pagefile.go
View File

@ -6,12 +6,13 @@ import (
"compress/gzip"
"encoding/binary"
"io"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/mdb/change"
"net"
"os"
"sync"
"time"
"git.crumpington.com/public/jldb/lib/errs"
"git.crumpington.com/public/jldb/mdb/change"
)
type File struct {

1
mdb/pfile/record_test.go
View File

@ -2,6 +2,7 @@ package pfile
import (
"bytes"
"git.crumpington.com/public/jldb/lib/wal"
"git.crumpington.com/public/jldb/mdb/change"
)

3
mdb/snapshot.go
View File

@ -3,8 +3,9 @@ package mdb
import (
"bytes"
"encoding/json"
"git.crumpington.com/public/jldb/mdb/change"
"sync/atomic"
"git.crumpington.com/public/jldb/mdb/change"
)
type Snapshot struct {

3
mdb/testing/crashconsistency/main.go
View File

@ -4,7 +4,6 @@ import (
"crypto/rand"
"errors"
"hash/crc32"
"git.crumpington.com/public/jldb/mdb"
"log"
mrand "math/rand"
"os"
@ -13,6 +12,8 @@ import (
"sync"
"sync/atomic"
"time"
"git.crumpington.com/public/jldb/mdb"
)
type DataItem struct {