package fstore import ( "bytes" "io" "net/http" "os" "path/filepath" "strconv" "sync" "time" "git.crumpington.com/public/jldb/lib/errs" "git.crumpington.com/public/jldb/lib/idgen" "git.crumpington.com/public/jldb/lib/rep" ) type Config struct { RootDir string Primary bool ReplicationPSK string NetTimeout time.Duration // Default is 1 minute. // WAL settings. WALSegMinCount int64 // Minimum Change sets in a segment. Default is 1024. WALSegMaxAgeSec int64 // Maximum age of a segment. Default is 1 hour. WALSegGCAgeSec int64 // Segment age for garbage collection. Default is 7 days. // For use by secondary. PrimaryEndpoint string } func (c Config) repConfig() rep.Config { return rep.Config{ RootDir: repDirPath(c.RootDir), Primary: c.Primary, ReplicationPSK: c.ReplicationPSK, NetTimeout: c.NetTimeout, WALSegMinCount: c.WALSegMinCount, WALSegMaxAgeSec: c.WALSegMaxAgeSec, WALSegGCAgeSec: c.WALSegGCAgeSec, PrimaryEndpoint: c.PrimaryEndpoint, SynchronousAppend: true, } } type Store struct { lock sync.Mutex buf *bytes.Buffer rep *rep.Replicator conf Config filesRoot string // Absolute, no trailing slash. tmpDir string // Absolute, no trailing slash. } func Open(conf Config) (*Store, error) { if conf.NetTimeout <= 0 { conf.NetTimeout = time.Minute } s := &Store{ buf: &bytes.Buffer{}, conf: conf, filesRoot: filesRootPath(conf.RootDir), tmpDir: tempDirPath(conf.RootDir), } var err error repConf := s.conf.repConfig() s.rep, err = rep.Open( rep.App{ SendState: s.repSendState, RecvState: s.repRecvState, InitStorage: s.repInitStorage, Replay: s.repReplay, LoadFromStorage: s.repLoadFromStorage, Apply: s.repApply, }, repConf) if err != nil { return nil, err } return s, nil } func (s *Store) GetTempFilePath() string { return tempFilePath(s.tmpDir, idgen.Next()) } func (s *Store) StoreString(str string, finalPath string) error { return s.StoreBytes([]byte(str), finalPath) } func (s *Store) StoreBytes(b []byte, finalPath string) error { tmpPath := s.GetTempFilePath() if err := os.WriteFile(tmpPath, b, 0600); err != nil { return err } defer os.RemoveAll(tmpPath) return s.Store(tmpPath, finalPath) } func (s *Store) Store(tmpPath, finalPath string) error { if !s.conf.Primary { return errs.NotAuthorized.WithMsg("not primary") } userPath, _, err := s.cleanAndValidatePath(finalPath) if err != nil { return err } idStr := filepath.Base(tmpPath) tmpID, _ := strconv.ParseUint(idStr, 10, 64) s.lock.Lock() defer s.lock.Unlock() f, err := os.Open(tmpPath) if err != nil { return err } defer f.Close() fi, err := f.Stat() if err != nil { return err } cmd := command{ Path: userPath, Store: true, TempID: tmpID, FileSize: fi.Size(), File: f, } size, reader := cmd.Reader(s.buf) _, _, err = s.rep.Append(size, reader) return err } func (s *Store) Remove(filePath string) error { if !s.conf.Primary { return errs.NotAuthorized.WithMsg("not primary") } userPath, _, err := s.cleanAndValidatePath(filePath) if err != nil { return err } s.lock.Lock() defer s.lock.Unlock() cmd := command{ Path: userPath, Store: false, TempID: 0, FileSize: 0, } size, reader := cmd.Reader(s.buf) _, _, err = s.rep.Append(size, reader) return err } func (s *Store) List(p string) (dirs, files []string, err error) { _, fullPath, err := s.cleanAndValidatePath(p) if err != nil { return nil, nil, err } fi, err := os.Stat(fullPath) if err != nil { if os.IsNotExist(err) { return nil, nil, nil } return nil, nil, err } if !fi.IsDir() { return nil, []string{fi.Name()}, nil } entries, err := os.ReadDir(fullPath) if err != nil { return nil, nil, err } for _, e := range entries { if e.IsDir() { dirs = append(dirs, e.Name()) } else { files = append(files, e.Name()) } } return dirs, files, nil } func (s *Store) Stat(p string) (os.FileInfo, error) { _, fullPath, err := s.cleanAndValidatePath(p) if err != nil { return nil, err } fi, err := os.Stat(fullPath) if err != nil { return nil, err } return fi, nil } func (s *Store) WriteTo(w io.Writer, filePath string) error { _, fullPath, err := s.cleanAndValidatePath(filePath) if err != nil { return err } f, err := os.Open(fullPath) if err != nil { return err } defer f.Close() if _, err := io.Copy(w, f); err != nil { return err } return nil } func (s *Store) Serve(w http.ResponseWriter, r *http.Request, p string) { _, fullPath, err := s.cleanAndValidatePath(p) if err != nil { http.Error(w, "not found", http.StatusNotFound) return } http.ServeFile(w, r, fullPath) } func (s *Store) ServeFallback(w http.ResponseWriter, r *http.Request, paths ...string) { for _, p := range paths { _, fullPath, err := s.cleanAndValidatePath(p) if err == nil { fi, err := os.Stat(fullPath) if err == nil && !fi.IsDir() { http.ServeFile(w, r, fullPath) return } } } http.Error(w, "not found", http.StatusNotFound) } func (s *Store) Handle(w http.ResponseWriter, r *http.Request) { s.rep.Handle(w, r) } func (s *Store) Close() error { return s.rep.Close() } func (s *Store) cleanAndValidatePath(in string) (userPath, fullPath string, err error) { userPath = cleanPath(in) if err := validatePath(userPath); err != nil { return "", "", err } return userPath, filepath.Join(s.filesRoot, userPath), nil }