package rep import ( "io" "git.crumpington.com/public/jldb/lib/atomicheader" "git.crumpington.com/public/jldb/lib/errs" "git.crumpington.com/public/jldb/lib/wal" "net" "os" "sync" "sync/atomic" "time" ) type Config struct { RootDir string Primary bool ReplicationPSK string NetTimeout time.Duration // Default is 1 minute. // WAL settings. WALSegMinCount int64 // Minimum Change sets in a segment. Default is 1024. WALSegMaxAgeSec int64 // Maximum age of a segment. Default is 1 hour. WALSegGCAgeSec int64 // Segment age for garbage collection. Default is 7 days. // If true, Append won't return until a successful App.Apply. SynchronousAppend bool // Necessary for secondary. PrimaryEndpoint string } type App struct { // SendState: The primary may need to send storage state to a secondary node. SendState func(conn net.Conn) error // (1) RecvState: Secondary nodes may need to load state from the primary if the // WAL is too far behind. RecvState func(conn net.Conn) error // (2) InitStorage: Prepare application storage for possible calls to // Replay. InitStorage func() error // (3) Replay: write the change to storage. Replay must be idempotent. Replay func(rec wal.Record) error // (4) LoadFromStorage: load the application's state from it's persistent // storage. LoadFromStorage func() error // (5) Apply: write the change to persistent storage. Apply must be // idempotent. In normal operation each change is applied exactly once. Apply func(rec wal.Record) error } type Replicator struct { app App conf Config lockFile *os.File pskBytes []byte wal *wal.WAL appendNotify chan struct{} // lock protects state. The lock is held when replaying (R), following (R), // and sending state (W). stateFile *os.File state *atomic.Pointer[localState] stateHandler *atomicheader.Handler stop chan struct{} done *sync.WaitGroup client *client // For secondary connection to primary. } func Open(app App, conf Config) (*Replicator, error) { rep := &Replicator{ app: app, conf: conf, state: &atomic.Pointer[localState]{}, stop: make(chan struct{}), done: &sync.WaitGroup{}, appendNotify: make(chan struct{}, 1), } rep.loadConfigDefaults() rep.state.Store(&localState{}) rep.client = newClient(rep.conf.PrimaryEndpoint, rep.conf.ReplicationPSK, rep.conf.NetTimeout) if err := rep.initDirectories(); err != nil { return nil, err } if err := rep.acquireLock(); err != nil { rep.Close() return nil, err } if err := rep.loadLocalState(); err != nil { rep.Close() return nil, err } if err := rep.openWAL(); err != nil { rep.Close() return nil, err } if err := rep.recvStateIfNecessary(); err != nil { rep.Close() return nil, err } if err := rep.app.InitStorage(); err != nil { rep.Close() return nil, err } if err := rep.replay(); err != nil { rep.Close() return nil, err } if err := rep.app.LoadFromStorage(); err != nil { rep.Close() return nil, err } rep.startWALGC() rep.startWALFollower() if !rep.conf.Primary { rep.startWALRecvr() } return rep, nil } func (rep *Replicator) Append(size int64, r io.Reader) (int64, int64, error) { if !rep.conf.Primary { return 0, 0, errs.NotAllowed.WithMsg("cannot write to secondary") } seqNum, timestampMS, err := rep.wal.Append(size, r) if err != nil { return 0, 0, err } if !rep.conf.SynchronousAppend { return seqNum, timestampMS, nil } <-rep.appendNotify return seqNum, timestampMS, nil } func (rep *Replicator) Primary() bool { return rep.conf.Primary } // TODO: Probably remove this. // The caller may call Ack after Apply to acknowledge that the change has also // been applied to the caller's application. Alternatively, the caller may use // follow to apply changes to their application state. func (rep *Replicator) ack(seqNum, timestampMS int64) error { state := rep.getState() state.SeqNum = seqNum state.TimestampMS = timestampMS return rep.setState(state) } func (rep *Replicator) getState() localState { return *rep.state.Load() } func (rep *Replicator) setState(state localState) error { err := rep.stateHandler.Write(func(page []byte) error { state.writeTo(page) return nil }) if err != nil { return err } rep.state.Store(&state) return nil } func (rep *Replicator) Info() Info { state := rep.getState() walInfo := rep.wal.Info() return Info{ AppSeqNum: state.SeqNum, AppTimestampMS: state.TimestampMS, WALFirstSeqNum: walInfo.FirstSeqNum, WALLastSeqNum: walInfo.LastSeqNum, WALLastTimestampMS: walInfo.LastTimestampMS, } } func (rep *Replicator) Close() error { if rep.stopped() { return nil } close(rep.stop) rep.done.Wait() if rep.lockFile != nil { rep.lockFile.Close() } if rep.wal != nil { rep.wal.Close() } if rep.client != nil { rep.client.Close() } return nil } func (rep *Replicator) stopped() bool { select { case <-rep.stop: return true default: return false } }