From 270347a05c84339c2c27dbb22b85a780e67a4547 Mon Sep 17 00:00:00 2001 From: jdl Date: Mon, 16 Oct 2023 10:45:53 +0200 Subject: [PATCH] WIP --- lib/rep/functions.go | 52 ---------- lib/rep/http-client.go | 170 ++++++++++----------------------- lib/rep/http-handler-util.go | 54 +++++++++++ lib/rep/http-handler.go | 113 ++++++++++------------ lib/rep/paths.go | 17 ++++ lib/rep/pools.go | 21 ---- lib/rep/psk.go | 16 ++++ lib/rep/replicator-open.go | 4 +- lib/rep/replicator-walrecvr.go | 7 +- lib/rep/replicator.go | 5 +- lib/rep/testapp_test.go | 3 +- 11 files changed, 196 insertions(+), 266 deletions(-) delete mode 100644 lib/rep/functions.go create mode 100644 lib/rep/http-handler-util.go create mode 100644 lib/rep/paths.go delete mode 100644 lib/rep/pools.go create mode 100644 lib/rep/psk.go diff --git a/lib/rep/functions.go b/lib/rep/functions.go deleted file mode 100644 index c7b18ea..0000000 --- a/lib/rep/functions.go +++ /dev/null @@ -1,52 +0,0 @@ -package rep - -import ( - "encoding/binary" - "encoding/json" - "net" - "path/filepath" - "time" - - "git.crumpington.com/public/jldb/lib/errs" -) - -// ---------------------------------------------------------------------------- - -func lockFilePath(rootDir string) string { - return filepath.Join(rootDir, "lock") -} - -func walRootDir(rootDir string) string { - return filepath.Join(rootDir, "wal") -} - -func stateFilePath(rootDir string) string { - return filepath.Join(rootDir, "state") -} - -// ---------------------------------------------------------------------------- - -func sendJSON( - item any, - conn net.Conn, - timeout time.Duration, -) error { - - buf := bufPoolGet() - defer bufPoolPut(buf) - - if err := json.NewEncoder(buf).Encode(item); err != nil { - return errs.Unexpected.WithErr(err) - } - - sizeBuf := make([]byte, 2) - binary.LittleEndian.PutUint16(sizeBuf, uint16(buf.Len())) - - conn.SetWriteDeadline(time.Now().Add(timeout)) - buffers := net.Buffers{sizeBuf, buf.Bytes()} - if _, err := buffers.WriteTo(conn); err != nil { - return errs.IO.WithErr(err) - } - - return nil -} diff --git a/lib/rep/http-client.go b/lib/rep/http-client.go index 19ded81..4b3fdbf 100644 --- a/lib/rep/http-client.go +++ b/lib/rep/http-client.go @@ -1,10 +1,10 @@ package rep import ( - "encoding/binary" "encoding/json" - "io" "net" + "net/http" + "strings" "sync" "time" @@ -14,166 +14,96 @@ import ( ) type client struct { - // Mutex-protected variables. - lock sync.Mutex - closed bool - conn net.Conn + client *http.Client // The following are constant. endpoint string - psk []byte + pskBytes [64]byte timeout time.Duration - buf []byte + lock sync.Mutex + conn net.Conn } func newClient(endpoint, psk string, timeout time.Duration) *client { - b := make([]byte, 256) - copy(b, []byte(psk)) + httpClient := &http.Client{ + Timeout: timeout, + } + + if !strings.HasSuffix(endpoint, "/") { + endpoint += "/" + } return &client{ + client: httpClient, endpoint: endpoint, - psk: b, + pskBytes: pskToBytes(psk), timeout: timeout, } } func (c *client) GetInfo() (info Info, err error) { - err = c.withConn(cmdGetInfo, func(conn net.Conn) error { - return c.recvJSON(&info, conn, c.timeout) - }) - return info, err + req, err := http.NewRequest(http.MethodGet, c.endpoint+pathGetInfo, nil) + if err != nil { + return info, errs.Unexpected.WithErr(err) + } + req.SetBasicAuth("", string(c.pskBytes[:])) + + resp, err := c.client.Do(req) + if err != nil { + return info, errs.IO.WithErr(err) + } + defer resp.Body.Close() + + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + return info, errs.IO.WithErr(err) + } + return info, nil } func (c *client) RecvState(recv func(net.Conn) error) error { - return c.withConn(cmdSendState, recv) + err := c.dialConnect(c.endpoint + pathSendState) + if err != nil { + return err + } + defer c.conn.Close() + + return recv(c.conn) } func (c *client) StreamWAL(w *wal.WAL) error { - return c.withConn(cmdStreamWAL, func(conn net.Conn) error { - return w.Recv(conn, c.timeout) - }) + err := c.dialConnect(c.endpoint + pathStreamWAL) + if err != nil { + return err + } + defer c.conn.Close() + + return w.Recv(c.conn, c.timeout) } func (c *client) Close() { c.lock.Lock() defer c.lock.Unlock() - c.closed = true - if c.conn != nil { c.conn.Close() - c.conn = nil } } // ---------------------------------------------------------------------------- -func (c *client) writeCmd(cmd byte) error { - c.conn.SetWriteDeadline(time.Now().Add(c.timeout)) - if _, err := c.conn.Write([]byte{cmd}); err != nil { - return errs.IO.WithErr(err) - } - return nil -} - -func (c *client) dial() error { - c.conn = nil - - conn, err := httpconn.Dial(c.endpoint) +func (c *client) dialConnect(endpoint string) error { + conn, err := httpconn.Dial(endpoint) if err != nil { return err } conn.SetWriteDeadline(time.Now().Add(c.timeout)) - if _, err := conn.Write(c.psk); err != nil { - conn.Close() + if _, err := conn.Write(c.pskBytes[:]); err != nil { return errs.IO.WithErr(err) } + c.lock.Lock() + defer c.lock.Unlock() c.conn = conn return nil } - -func (c *client) withConn(cmd byte, fn func(net.Conn) error) error { - conn, err := c.getConn(cmd) - if err != nil { - return err - } - - if err := fn(conn); err != nil { - conn.Close() - return err - } - return nil -} - -func (c *client) getConn(cmd byte) (net.Conn, error) { - c.lock.Lock() - defer c.lock.Unlock() - - if c.closed { - return nil, errs.IO.WithErr(io.EOF) - } - - dialed := false - - if c.conn == nil { - if err := c.dial(); err != nil { - return nil, err - } - dialed = true - } - - if err := c.writeCmd(cmd); err != nil { - if dialed { - c.conn = nil - return nil, err - } - - if err := c.dial(); err != nil { - return nil, err - } - - if err := c.writeCmd(cmd); err != nil { - return nil, err - } - } - - return c.conn, nil -} - -func (c *client) recvJSON( - item any, - conn net.Conn, - timeout time.Duration, -) error { - - if cap(c.buf) < 2 { - c.buf = make([]byte, 0, 1024) - } - buf := c.buf[:2] - - conn.SetReadDeadline(time.Now().Add(timeout)) - - if _, err := io.ReadFull(conn, buf); err != nil { - return errs.IO.WithErr(err) - } - - size := binary.LittleEndian.Uint16(buf) - - if cap(buf) < int(size) { - buf = make([]byte, size) - c.buf = buf - } - buf = buf[:size] - - if _, err := io.ReadFull(conn, buf); err != nil { - return errs.IO.WithErr(err) - } - - if err := json.Unmarshal(buf, item); err != nil { - return errs.Unexpected.WithErr(err) - } - - return nil -} diff --git a/lib/rep/http-handler-util.go b/lib/rep/http-handler-util.go new file mode 100644 index 0000000..05d4877 --- /dev/null +++ b/lib/rep/http-handler-util.go @@ -0,0 +1,54 @@ +package rep + +import ( + "crypto/subtle" + "io" + "log" + "net" + "net/http" + "time" + + "git.crumpington.com/public/jldb/lib/httpconn" +) + +func (rep *Replicator) handlerLogf(pattern string, args ...any) { + log.Printf("[HTTP-HANDLER] "+pattern, args...) +} + +// checkBasicAuth tests whether the caller has provided the appropriate basic auth +// header. The caller should provide the PSK as the basic-auth password. +func (rep *Replicator) checkBasicAuth(w http.ResponseWriter, r *http.Request) bool { + _, pwd, _ := r.BasicAuth() + if subtle.ConstantTimeCompare([]byte(pwd), rep.pskBytes[:]) != 1 { + rep.handlerLogf("PSK mismatch.") + http.Error(w, "not authorized", http.StatusUnauthorized) + return false + } + return true +} + +// acceptConnect accepts a CONNECT request and checks the PSK. +func (rep *Replicator) acceptConnect(w http.ResponseWriter, r *http.Request) net.Conn { + conn, err := httpconn.Accept(w, r) + if err != nil { + rep.handlerLogf("Failed to accept connection: %s", err) + return nil + } + + psk := [64]byte{} + + conn.SetReadDeadline(time.Now().Add(rep.conf.NetTimeout)) + if _, err := io.ReadFull(conn, psk[:]); err != nil { + conn.Close() + rep.handlerLogf("Failed to read PSK: %v", err) + return nil + } + + if subtle.ConstantTimeCompare(psk[:], rep.pskBytes[:]) != 1 { + conn.Close() + rep.handlerLogf("PSK mismatch.") + return nil + } + + return conn +} diff --git a/lib/rep/http-handler.go b/lib/rep/http-handler.go index 293ffb7..843cb7a 100644 --- a/lib/rep/http-handler.go +++ b/lib/rep/http-handler.go @@ -1,82 +1,69 @@ package rep import ( - "crypto/subtle" - "log" + "encoding/json" "net/http" - "time" - - "git.crumpington.com/public/jldb/lib/httpconn" + "path" ) const ( - cmdGetInfo = 10 - cmdSendState = 20 - cmdStreamWAL = 30 + pathGetInfo = "get-info" + pathSendState = "send-state" + pathStreamWAL = "stream-wal" ) -// --------------------------------------------------------------------------- - func (rep *Replicator) Handle(w http.ResponseWriter, r *http.Request) { - logf := func(pattern string, args ...any) { - log.Printf("[HTTP-HANDLER] "+pattern, args...) + // We'll handle two types of requests: HTTP GET requests for JSON, or + // streaming requets for state or wall. + + base := path.Base(r.URL.Path) + switch base { + case pathGetInfo: + rep.handleGetInfo(w, r) + case pathSendState: + rep.handleSendState(w, r) + case pathStreamWAL: + rep.handleStreamWAL(w, r) + default: + http.Error(w, "not found", http.StatusNotFound) + } +} + +func (rep *Replicator) handleGetInfo(w http.ResponseWriter, r *http.Request) { + if !rep.checkBasicAuth(w, r) { + return } - conn, err := httpconn.Accept(w, r) - if err != nil { - logf("Failed to accept connection: %s", err) + w.Header().Set("Content-Type", "application/json") + + if err := json.NewEncoder(w).Encode(rep.Info()); err != nil { + rep.handlerLogf("Failed to send info: %s", err) + } +} + +func (rep *Replicator) handleSendState(w http.ResponseWriter, r *http.Request) { + conn := rep.acceptConnect(w, r) + if conn == nil { return } defer conn.Close() - psk := make([]byte, 256) - - conn.SetReadDeadline(time.Now().Add(rep.conf.NetTimeout)) - if _, err := conn.Read(psk); err != nil { - logf("Failed to read PSK: %v", err) - return - } - - expected := rep.pskBytes - if subtle.ConstantTimeCompare(expected, psk) != 1 { - logf("PSK mismatch.") - return - } - - cmd := make([]byte, 1) - - for { - conn.SetReadDeadline(time.Now().Add(rep.conf.NetTimeout)) - if _, err := conn.Read(cmd); err != nil { - logf("Read failed: %v", err) - return - } - - switch cmd[0] { - - case cmdGetInfo: - - if err := sendJSON(rep.Info(), conn, rep.conf.NetTimeout); err != nil { - logf("Failed to send info: %s", err) - return - } - - case cmdSendState: - - if err := rep.sendState(conn); err != nil { - if !rep.stopped() { - logf("Failed to send state: %s", err) - } - return - } - - case cmdStreamWAL: - - err := rep.wal.Send(conn, rep.conf.NetTimeout) - if !rep.stopped() { - logf("Failed when sending WAL: %s", err) - } - return + if err := rep.sendState(conn); err != nil { + if !rep.stopped() { + rep.handlerLogf("Failed to send state: %s", err) } } } + +func (rep *Replicator) handleStreamWAL(w http.ResponseWriter, r *http.Request) { + conn := rep.acceptConnect(w, r) + if conn == nil { + return + } + defer conn.Close() + + err := rep.wal.Send(conn, rep.conf.NetTimeout) + if !rep.stopped() { + rep.handlerLogf("Failed when streaming WAL: %s", err) + } +} diff --git a/lib/rep/paths.go b/lib/rep/paths.go new file mode 100644 index 0000000..4241832 --- /dev/null +++ b/lib/rep/paths.go @@ -0,0 +1,17 @@ +package rep + +import ( + "path/filepath" +) + +func lockFilePath(rootDir string) string { + return filepath.Join(rootDir, "lock") +} + +func walRootDir(rootDir string) string { + return filepath.Join(rootDir, "wal") +} + +func stateFilePath(rootDir string) string { + return filepath.Join(rootDir, "state") +} diff --git a/lib/rep/pools.go b/lib/rep/pools.go deleted file mode 100644 index e539223..0000000 --- a/lib/rep/pools.go +++ /dev/null @@ -1,21 +0,0 @@ -package rep - -import ( - "bytes" - "sync" -) - -var bufPool = sync.Pool{ - New: func() any { - return &bytes.Buffer{} - }, -} - -func bufPoolGet() *bytes.Buffer { - return bufPool.Get().(*bytes.Buffer) -} - -func bufPoolPut(b *bytes.Buffer) { - b.Reset() - bufPool.Put(b) -} diff --git a/lib/rep/psk.go b/lib/rep/psk.go new file mode 100644 index 0000000..617ac2a --- /dev/null +++ b/lib/rep/psk.go @@ -0,0 +1,16 @@ +package rep + +import ( + "crypto/sha256" + "encoding/hex" +) + +func pskToBytes(in string) [64]byte { + b := sha256.Sum256([]byte(in)) + dst := [64]byte{} + i := hex.Encode(dst[:], b[:]) + if i != 64 { + panic(i) + } + return dst +} diff --git a/lib/rep/replicator-open.go b/lib/rep/replicator-open.go index f3ad18a..57f1a5e 100644 --- a/lib/rep/replicator-open.go +++ b/lib/rep/replicator-open.go @@ -27,9 +27,7 @@ func (rep *Replicator) loadConfigDefaults() { } rep.conf = conf - - rep.pskBytes = make([]byte, 256) - copy(rep.pskBytes, []byte(conf.ReplicationPSK)) + rep.pskBytes = pskToBytes(conf.ReplicationPSK) } func (rep *Replicator) initDirectories() error { diff --git a/lib/rep/replicator-walrecvr.go b/lib/rep/replicator-walrecvr.go index a5b5b05..039282e 100644 --- a/lib/rep/replicator-walrecvr.go +++ b/lib/rep/replicator-walrecvr.go @@ -30,9 +30,8 @@ func (rep *Replicator) runWALRecvrOnce() { log.Printf("[WAL-RECVR] "+pattern, args...) } - if err := rep.client.StreamWAL(rep.wal); err != nil { - if !rep.stopped() { - logf("Recv failed: %v", err) - } + err := rep.client.StreamWAL(rep.wal) + if !rep.stopped() { + logf("Recv failed: %v", err) } } diff --git a/lib/rep/replicator.go b/lib/rep/replicator.go index c4b611f..43bda0b 100644 --- a/lib/rep/replicator.go +++ b/lib/rep/replicator.go @@ -60,8 +60,9 @@ type Replicator struct { conf Config lockFile *os.File - pskBytes []byte - wal *wal.WAL + pskBytes [64]byte // 64 ascii characters. See pskToBytes. + + wal *wal.WAL appendNotify chan struct{} diff --git a/lib/rep/testapp_test.go b/lib/rep/testapp_test.go index b97baf9..163b7c6 100644 --- a/lib/rep/testapp_test.go +++ b/lib/rep/testapp_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "git.crumpington.com/public/jldb/lib/errs" "git.crumpington.com/public/jldb/lib/wal" ) @@ -83,7 +84,7 @@ func newApp(t *testing.T, id int64, conf Config) *TestApp { Apply: a.apply, }, conf) if err != nil { - t.Fatal(err) + t.Fatal(errs.FmtDetails(err)) } return a