fts5

fts5/fts5.go

@@ -0,0 +1,151 @@
+package fts5
+
+import (
+	"database/sql"
+	"fmt"
+	"iter"
+	"strings"
+
+	_ "github.com/mattn/go-sqlite3"
+)
+
+type FTS5 struct {
+	colNames []string
+	db       *sql.DB
+
+	insertStmt *sql.Stmt
+	matchStmt  *sql.Stmt
+}
+
+func OpenMem(columnNames ...string) (*FTS5, error) {
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		return nil, err
+	}
+	return newInternal(db, columnNames)
+}
+
+func Open(path string, columnNames ...string) (*FTS5, error) {
+	db, err := sql.Open("sqlite3", path)
+	if err != nil {
+		return nil, err
+	}
+	return newInternal(db, columnNames)
+}
+
+func newInternal(db *sql.DB, columnNames []string) (*FTS5, error) {
+	_, err := db.Exec(fmt.Sprintf(
+		`CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5(%s, tokenize='porter')`,
+		strings.Join(columnNames, ",")))
+	if err != nil {
+		db.Close()
+		return nil, err
+	}
+
+	insertStmt, err := prepareInsertStmt(db, columnNames)
+	if err != nil {
+		db.Close()
+		return nil, err
+	}
+
+	matchStmt, err := db.Prepare(`SELECT rowid,rank FROM search WHERE search=? ` +
+		`ORDER BY rank ` +
+		`LIMIT ? OFFSET ?`)
+	if err != nil {
+		db.Close()
+		return nil, err
+	}
+
+	return &FTS5{
+		colNames:   columnNames,
+		db:         db,
+		insertStmt: insertStmt,
+		matchStmt:  matchStmt,
+	}, nil
+}
+
+func (fts *FTS5) Close() error {
+	return fts.db.Close()
+}
+
+func (fts *FTS5) Upsert(id int64, data map[string]string) error {
+	return execInsertStmt(fts.insertStmt, fts.colNames, id, data)
+}
+
+type Item struct {
+	ID   int64
+	Data map[string]string
+}
+
+func (fts *FTS5) UpsertBulk(src iter.Seq2[Item, error]) error {
+	tx, err := fts.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	insertStmt, err := prepareInsertStmt(tx, fts.colNames)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+
+	for item, err := range src {
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+
+		if err := execInsertStmt(insertStmt, fts.colNames, item.ID, item.Data); err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+
+	return tx.Commit()
+}
+
+func (fts *FTS5) Delete(id int64) (bool, error) {
+	result, err := fts.db.Exec(`DELETE FROM search WHERE rowid=?`, id)
+	if err != nil {
+		return false, err
+	}
+
+	count, err := result.RowsAffected()
+	if err != nil {
+		panic(err)
+	}
+	if count > 1 {
+		panic("multiple rows deleted")
+	}
+	return count > 0, nil
+}
+
+func (fts *FTS5) DeleteAll() error {
+	_, err := fts.db.Exec(`DELETE FROM search`)
+	return err
+}
+
+type Result struct {
+	ID   int64
+	Rank float64
+}
+
+func (fts *FTS5) Match(query string, limit, offset int) ([]Result, error) {
+	results := make([]Result, 0, limit)
+
+	rows, err := fts.matchStmt.Query(query, limit, offset)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		result := Result{}
+		if err := rows.Scan(&result.ID, &result.Rank); err != nil {
+			return nil, err
+		}
+		results = append(results, result)
+	}
+
+	return results, nil
+}

fts5/fts5_test.go

@@ -0,0 +1,59 @@
+package fts5
+
+import (
+	"testing"
+)
+
+func TestMatch(t *testing.T) {
+	fts, err := OpenMem("Title", "Text")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fts.Upsert(100, map[string]string{
+		"Title": "Peter Rabbit",
+		"Text":  "Peter Rabbit is a fictional animal character in various children's stories by English author Beatrix Potter.",
+	})
+
+	fts.Upsert(200, map[string]string{
+		"Title": "Baloo",
+		"Text":  "Fictional bear from the Jungle Book.",
+	})
+
+	fts.Upsert(300, map[string]string{
+		"Title": "Barney Bear",
+		"Text":  "A grumpy brown bear.",
+	})
+
+	fts.Upsert(400, map[string]string{
+		"Title": "Peter the Panda",
+		"Text":  "A secret agent panda from Seattle.",
+	})
+
+	// Should have 2 results for "peter".
+	results, err := fts.Match("peter", 3, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(results) != 2 {
+		t.Fatal(results)
+	}
+
+	// Try limit.
+	results, err = fts.Match("peter", 1, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(results) != 1 {
+		t.Fatal(results)
+	}
+
+	// Try offset.
+	results, err = fts.Match("peter", 3, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(results) != 1 {
+		t.Fatal(results)
+	}
+}

fts5/query.go

@@ -0,0 +1,36 @@
+package fts5
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+)
+
+type preparer interface {
+	Prepare(string) (*sql.Stmt, error)
+}
+
+func prepareInsertStmt(
+	db preparer,
+	columnNames []string,
+) (*sql.Stmt, error) {
+	insertQuery := fmt.Sprintf(`INSERT INTO search(rowid,%s) VALUES (?%s)`,
+		strings.Join(columnNames, ","),
+		strings.Repeat(",?", len(columnNames)))
+	return db.Prepare(insertQuery)
+}
+
+func execInsertStmt(
+	stmt *sql.Stmt,
+	colNames []string,
+	id int64,
+	data map[string]string,
+) error {
+	values := make([]any, len(colNames)+1)
+	values[0] = id
+	for i, col := range colNames {
+		values[i+1] = data[col]
+	}
+	_, err := stmt.Exec(values...)
+	return err
+}

fts5/requiretag.go

@@ -0,0 +1,9 @@
+//go:build !fts5
+
+package fts5
+
+import "log"
+
+func init() {
+	log.Fatal("You must provide build tag `-tags fts5`.")
+}

sqlgen/gen.go.tmpl

@@ -5,24 +5,6 @@ import (
 	"iter"
 )
 
-var (
-	ErrConstraint = errors.New("constraint violation")
-	ErrNotFound = errors.New("not found")
-)
-
-func translateError(err error) error {
-	if err == nil {
-		return nil
-	}
-	if e, ok := err.(sqlite3.Error); ok && e.Code == 19 {
-		return ErrConstraint
-	}
-	if errors.Is(err, sql.ErrNoRows) {
-		return ErrNotFound
-	}
-	return err
-}
-
 type TX interface {
 	Exec(query string, args ...any) (sql.Result, error)
 	Query(query string, args ...any) (*sql.Rows, error)
@@ -54,7 +36,7 @@ func {{.Type}}_Insert(
   }
 
 	_, err = tx.Exec("{{.InsertQuery}}", {{.InsertArgs}})
-	return translateError(err)
+	return err
 }
 
 {{- end}} {{/* if not .NoInsert */}}
@@ -74,7 +56,7 @@ func {{.Type}}_Update(
 
 	result, err := tx.Exec("{{.UpdateQuery}}", {{.UpdateArgs}})
 	if err != nil {
-		return translateError(err)
+		return err
 	}
 
 	n, err := result.RowsAffected()
@@ -83,7 +65,7 @@ func {{.Type}}_Update(
 	}
 	switch n {
 	case 0:
-		return ErrNotFound
+		return sql.ErrNoRows
 	case 1:
 		return nil
 	default:
@@ -105,7 +87,7 @@ func {{.Type}}_UpdateFull(
 
 	result, err := tx.Exec("{{.UpdateFullQuery}}", {{.UpdateFullArgs}})
 	if err != nil {
-		return translateError(err)
+		return err
 	}
 
 	n, err := result.RowsAffected()
@@ -114,7 +96,7 @@ func {{.Type}}_UpdateFull(
 	}
 	switch n {
 		case 0:
-		return ErrNotFound
+		return sql.ErrNoRows
 	case 1:
 		return nil
 	default:
@@ -134,7 +116,7 @@ func {{.Type}}_Delete(
 ) (err error) {
 	result, err := tx.Exec("{{.DeleteQuery}}", {{.DeleteArgs}})
 	if err != nil {
-		return translateError(err)
+		return err
 	}
 
 	n, err := result.RowsAffected()
@@ -143,7 +125,7 @@ func {{.Type}}_Delete(
 	}
 	switch n {
 		case 0:
-		return ErrNotFound
+		return sql.ErrNoRows
 	case 1:
 		return nil
 	default:
@@ -162,7 +144,7 @@ func {{.Type}}_Get(
 ) {
   row = &{{.Type}}{}
 	r := tx.QueryRow("{{.GetQuery}}", {{.DeleteArgs}})
-	err = translateError(r.Scan({{.ScanArgs}}))
+	err = r.Scan({{.ScanArgs}})
 	return
 }
 
@@ -177,7 +159,7 @@ func {{.Type}}_GetWhere(
 ) {
   row = &{{.Type}}{}
 	r := tx.QueryRow(query, args...)
-	err = translateError(r.Scan({{.ScanArgs}}))
+	err = r.Scan({{.ScanArgs}})
 	return
 }
 
@@ -201,7 +183,7 @@ func {{.Type}}_Iterate(
 		defer rows.Close()
 		for rows.Next() {
 			row := &{{.Type}}{}
-			err := translateError(rows.Scan({{.ScanArgs}}))
+			err := rows.Scan({{.ScanArgs}})
 			if !yield(row, err) {
 				return
 			}

sqlgen/main.go

@@ -5,14 +5,19 @@ import (
 	"os"
 )
 
+const (
+	DRIVER_SQLITE   = "sqlite"
+	DRIVER_POSTGRES = "postgres"
+)
+
 func Main() {
 	usage := func() {
 		fmt.Fprintf(os.Stderr, `
 %s DRIVER DEFS_PATH OUTPUT_PATH
 
-Drivers are one of: sqlite, postgres
+Drivers are one of: %s %s
 `,
-			os.Args[0])
+			os.Args[0], DRIVER_SQLITE, DRIVER_POSTGRES)
 		os.Exit(1)
 	}
 
@@ -21,21 +26,20 @@ Drivers are one of: sqlite, postgres
 	}
 
 	var (
-		template   string
 		driver     = os.Args[1]
 		defsPath   = os.Args[2]
 		outputPath = os.Args[3]
 	)
 
 	switch driver {
-	case "sqlite":
-		template = sqliteTemplate
+	case DRIVER_SQLITE, DRIVER_POSTGRES:
+		// OK
 	default:
 		fmt.Fprintf(os.Stderr, "Unknown driver: %s", driver)
 		usage()
 	}
 
-	err := render(template, defsPath, outputPath)
+	err := render(driver, defsPath, outputPath)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v", err)
 		os.Exit(1)

sqlgen/parse.go

@@ -6,15 +6,15 @@ import (
 	"strings"
 )
 
-func parsePath(filePath string) (*schema, error) {
+func parsePath(driver, filePath string) (*schema, error) {
 	fileBytes, err := os.ReadFile(filePath)
 	if err != nil {
 		return nil, err
 	}
-	return parseBytes(fileBytes)
+	return parseBytes(driver, fileBytes)
 }
 
-func parseBytes(fileBytes []byte) (*schema, error) {
+func parseBytes(driver string, fileBytes []byte) (*schema, error) {
 	s := string(fileBytes)
 	for _, c := range []string{",", "(", ")", ";"} {
 		s = strings.ReplaceAll(s, c, " "+c+" ")
@@ -29,7 +29,7 @@ func parseBytes(fileBytes []byte) (*schema, error) {
 	for len(tokens) > 0 {
 		switch tokens[0] {
 		case "TABLE":
-			tokens, err = parseTable(schema, tokens)
+			tokens, err = parseTable(driver, schema, tokens)
 			if err != nil {
 				return nil, err
 			}
@@ -42,7 +42,7 @@ func parseBytes(fileBytes []byte) (*schema, error) {
 	return schema, nil
 }
 
-func parseTable(schema *schema, tokens []string) ([]string, error) {
+func parseTable(driver string, schema *schema, tokens []string) ([]string, error) {
 	tokens = tokens[1:]
 	if len(tokens) < 3 {
 		return tokens, errors.New("incomplete table definition")
@@ -52,6 +52,7 @@ func parseTable(schema *schema, tokens []string) ([]string, error) {
 	}
 
 	table := &table{
+		driver: driver,
 		Name:   tokens[0],
 		Type:   tokens[2],
 	}

sqlgen/parse_test.go

@@ -22,7 +22,7 @@ func TestParse(t *testing.T) {
 
 	for _, defPath := range paths {
 		t.Run(filepath.Base(defPath), func(t *testing.T) {
-			parsed, err := parsePath(defPath)
+			parsed, err := parsePath("", defPath)
 			if err != nil {
 				t.Fatal(err)
 			}

sqlgen/schema.go

@@ -10,6 +10,7 @@ type schema struct {
 }
 
 type table struct {
+	driver   string //
 	Name     string // Name in SQL
 	Type     string // Go type
 	NoInsert bool
@@ -29,6 +30,19 @@ type column struct {
 
 // ----------------------------------------------------------------------------
 
+func (t *table) translateQuery(in string) string {
+	if t.driver == DRIVER_SQLITE {
+		return in
+	}
+
+	i := 1
+	for strings.Contains(in, "?") {
+		in = strings.Replace(in, "?", fmt.Sprintf("$%d", i), 1)
+		i++
+	}
+	return in
+}
+
 func (t *table) colSQLNames() []string {
 	names := make([]string, len(t.Columns))
 	for i := range names {
@@ -72,7 +86,7 @@ func (t *table) InsertQuery() string {
 		b.WriteString(`?`)
 	}
 	b.WriteString(`)`)
-	return b.String()
+	return t.translateQuery(b.String())
 }
 
 func (t *table) InsertArgs() string {
@@ -116,7 +130,7 @@ func (t *table) UpdateQuery() string {
 		b.WriteString(` ` + c.SqlName + `=?`)
 	}
 
-	return b.String()
+	return t.translateQuery(b.String())
 }
 
 func (t *table) UpdateArgs() string {
@@ -166,7 +180,7 @@ func (t *table) UpdateFullQuery() string {
 		b.WriteString(` ` + c.SqlName + `=?`)
 	}
 
-	return b.String()
+	return t.translateQuery(b.String())
 }
 
 func (t *table) UpdateFullArgs() string {
@@ -222,7 +236,7 @@ func (t *table) DeleteQuery() string {
 		b.WriteString(col.SqlName)
 		b.WriteString(`=?`)
 	}
-	return b.String()
+	return t.translateQuery(b.String())
 }
 
 func (t *table) DeleteArgs() string {
@@ -248,7 +262,7 @@ func (t *table) GetQuery() string {
 		}
 		b.WriteString(col.SqlName + `=?`)
 	}
-	return b.String()
+	return t.translateQuery(b.String())
 }
 
 func (t *table) ScanArgs() string {

sqlgen/template.go

@@ -8,16 +8,16 @@ import (
 	"text/template"
 )
 
-//go:embed sqlite.go.tmpl
-var sqliteTemplate string
+//go:embed gen.go.tmpl
+var fileTemplate string
 
-func render(templateStr, schemaPath, outputPath string) error {
-	sch, err := parsePath(schemaPath)
+func render(driver, schemaPath, outputPath string) error {
+	sch, err := parsePath(driver, schemaPath)
 	if err != nil {
 		return err
 	}
 
-	tmpl := template.Must(template.New("").Parse(templateStr))
+	tmpl := template.Must(template.New("").Parse(fileTemplate))
 	fOut, err := os.Create(outputPath)
 	if err != nil {
 		return err