jldb/fstore/store.go

package fstore

import (
	"bytes"
	"io"
	"git.crumpington.com/public/jldb/lib/errs"
	"git.crumpington.com/public/jldb/lib/idgen"
	"git.crumpington.com/public/jldb/lib/rep"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"time"
)

type Config struct {
	RootDir        string
	Primary        bool
	ReplicationPSK string
	NetTimeout     time.Duration // Default is 1 minute.

	// WAL settings.
	WALSegMinCount  int64 // Minimum Change sets in a segment. Default is 1024.
	WALSegMaxAgeSec int64 // Maximum age of a segment. Default is 1 hour.
	WALSegGCAgeSec  int64 // Segment age for garbage collection. Default is 7 days.

	// For use by secondary.
	PrimaryEndpoint string
}

func (c Config) repConfig() rep.Config {
	return rep.Config{
		RootDir:           repDirPath(c.RootDir),
		Primary:           c.Primary,
		ReplicationPSK:    c.ReplicationPSK,
		NetTimeout:        c.NetTimeout,
		WALSegMinCount:    c.WALSegMinCount,
		WALSegMaxAgeSec:   c.WALSegMaxAgeSec,
		WALSegGCAgeSec:    c.WALSegGCAgeSec,
		PrimaryEndpoint:   c.PrimaryEndpoint,
		SynchronousAppend: true,
	}
}

type Store struct {
	lock      sync.Mutex
	buf       *bytes.Buffer
	rep       *rep.Replicator
	conf      Config
	filesRoot string // Absolute, no trailing slash.
	tmpDir    string // Absolute, no trailing slash.
}

func Open(conf Config) (*Store, error) {
	if conf.NetTimeout <= 0 {
		conf.NetTimeout = time.Minute
	}

	s := &Store{
		buf:       &bytes.Buffer{},
		conf:      conf,
		filesRoot: filesRootPath(conf.RootDir),
		tmpDir:    tempDirPath(conf.RootDir),
	}

	var err error

	repConf := s.conf.repConfig()

	s.rep, err = rep.Open(
		rep.App{
			SendState:       s.repSendState,
			RecvState:       s.repRecvState,
			InitStorage:     s.repInitStorage,
			Replay:          s.repReplay,
			LoadFromStorage: s.repLoadFromStorage,
			Apply:           s.repApply,
		},
		repConf)

	if err != nil {
		return nil, err
	}

	return s, nil
}

func (s *Store) GetTempFilePath() string {
	return tempFilePath(s.tmpDir, idgen.Next())
}

func (s *Store) StoreString(str string, finalPath string) error {
	return s.StoreBytes([]byte(str), finalPath)
}

func (s *Store) StoreBytes(b []byte, finalPath string) error {
	tmpPath := s.GetTempFilePath()

	if err := os.WriteFile(tmpPath, b, 0600); err != nil {
		return err
	}
	defer os.RemoveAll(tmpPath)

	return s.Store(tmpPath, finalPath)
}

func (s *Store) Store(tmpPath, finalPath string) error {
	if !s.conf.Primary {
		return errs.NotAuthorized.WithMsg("not primary")
	}

	userPath, _, err := s.cleanAndValidatePath(finalPath)
	if err != nil {
		return err
	}

	idStr := filepath.Base(tmpPath)
	tmpID, _ := strconv.ParseUint(idStr, 10, 64)

	s.lock.Lock()
	defer s.lock.Unlock()

	f, err := os.Open(tmpPath)
	if err != nil {
		return err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return err
	}

	cmd := command{
		Path:     userPath,
		Store:    true,
		TempID:   tmpID,
		FileSize: fi.Size(),
		File:     f,
	}

	size, reader := cmd.Reader(s.buf)

	_, _, err = s.rep.Append(size, reader)
	return err
}

func (s *Store) Remove(filePath string) error {
	if !s.conf.Primary {
		return errs.NotAuthorized.WithMsg("not primary")
	}

	userPath, _, err := s.cleanAndValidatePath(filePath)
	if err != nil {
		return err
	}

	s.lock.Lock()
	defer s.lock.Unlock()

	cmd := command{
		Path:     userPath,
		Store:    false,
		TempID:   0,
		FileSize: 0,
	}

	size, reader := cmd.Reader(s.buf)

	_, _, err = s.rep.Append(size, reader)
	return err
}

func (s *Store) List(p string) (dirs, files []string, err error) {
	_, fullPath, err := s.cleanAndValidatePath(p)
	if err != nil {
		return nil, nil, err
	}

	fi, err := os.Stat(fullPath)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil, nil
		}
		return nil, nil, err
	}

	if !fi.IsDir() {
		return nil, []string{fi.Name()}, nil
	}

	entries, err := os.ReadDir(fullPath)
	if err != nil {
		return nil, nil, err
	}

	for _, e := range entries {
		if e.IsDir() {
			dirs = append(dirs, e.Name())
		} else {
			files = append(files, e.Name())
		}
	}

	return dirs, files, nil
}

func (s *Store) Stat(p string) (os.FileInfo, error) {
	_, fullPath, err := s.cleanAndValidatePath(p)
	if err != nil {
		return nil, err
	}

	fi, err := os.Stat(fullPath)
	if err != nil {
		return nil, err
	}

	return fi, nil
}

func (s *Store) WriteTo(w io.Writer, filePath string) error {
	_, fullPath, err := s.cleanAndValidatePath(filePath)
	if err != nil {
		return err
	}

	f, err := os.Open(fullPath)
	if err != nil {
		return err
	}
	defer f.Close()

	if _, err := io.Copy(w, f); err != nil {
		return err
	}

	return nil
}

func (s *Store) Serve(w http.ResponseWriter, r *http.Request, p string) {
	_, fullPath, err := s.cleanAndValidatePath(p)
	if err != nil {
		http.Error(w, "not found", http.StatusNotFound)
		return
	}
	http.ServeFile(w, r, fullPath)
}

func (s *Store) ServeFallback(w http.ResponseWriter, r *http.Request, paths ...string) {
	for _, p := range paths {
		_, fullPath, err := s.cleanAndValidatePath(p)
		if err == nil {
			fi, err := os.Stat(fullPath)
			if err == nil && !fi.IsDir() {
				http.ServeFile(w, r, fullPath)
				return
			}
		}
	}
	http.Error(w, "not found", http.StatusNotFound)
}

func (s *Store) Handle(w http.ResponseWriter, r *http.Request) {
	s.rep.Handle(w, r)
}

func (s *Store) Close() error {
	return s.rep.Close()
}

func (s *Store) cleanAndValidatePath(in string) (userPath, fullPath string, err error) {
	userPath = cleanPath(in)
	if err := validatePath(userPath); err != nil {
		return "", "", err
	}
	return userPath, filepath.Join(s.filesRoot, userPath), nil
}