Feat: add rag [wip; skip-ci]

storage/migrate.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"io/fs"
 	"strings"
+
+	_ "github.com/asg017/sqlite-vec-go-bindings/ncruces"
 )
 
 //go:embed migrations/*
@@ -27,6 +29,7 @@ func (p *ProviderSQL) Migrate() {
 			err := p.executeMigration(migrationsDir, file.Name())
 			if err != nil {
 				p.logger.Error("Failed to execute migration %s: %v", file.Name(), err)
+				panic(err)
 			}
 		}
 	}
@@ -51,7 +54,7 @@ func (p *ProviderSQL) executeMigration(migrationsDir fs.FS, fileName string) err
 
 func (p *ProviderSQL) executeSQL(sqlContent []byte) error {
 	// Connect to the database (example using a simple connection)
-	_, err := p.db.Exec(string(sqlContent))
+	err := p.s3Conn.Exec(string(sqlContent))
 	if err != nil {
 		return fmt.Errorf("failed to execute SQL: %w", err)
 	}

storage/migrations/002_add_vector.up.sql

@@ -0,0 +1,6 @@
+CREATE VIRTUAL TABLE IF NOT EXISTS embeddings USING vec0(
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    embedding FLOAT[5120],
+    slug TEXT NOT NULL,
+    raw_text TEXT NOT NULL
+);

storage/storage.go

@@ -6,11 +6,13 @@ import (
 
 	_ "github.com/glebarez/go-sqlite"
 	"github.com/jmoiron/sqlx"
+	"github.com/ncruces/go-sqlite3"
 )
 
 type FullRepo interface {
 	ChatHistory
 	Memories
+	VectorRepo
 }
 
 type ChatHistory interface {
@@ -25,6 +27,7 @@ type ChatHistory interface {
 
 type ProviderSQL struct {
 	db     *sqlx.DB
+	s3Conn *sqlite3.Conn
 	logger *slog.Logger
 }
 
@@ -87,6 +90,7 @@ func (p ProviderSQL) ChatGetMaxID() (uint32, error) {
 	return id, err
 }
 
+// opens two connections
 func NewProviderSQL(dbPath string, logger *slog.Logger) FullRepo {
 	db, err := sqlx.Open("sqlite", dbPath)
 	if err != nil {
@@ -94,6 +98,11 @@ func NewProviderSQL(dbPath string, logger *slog.Logger) FullRepo {
 		return nil
 	}
 	p := ProviderSQL{db: db, logger: logger}
+	p.s3Conn, err = sqlite3.Open(dbPath)
+	if err != nil {
+		logger.Error("failed to open vecdb connection", "error", err)
+		return nil
+	}
 	p.Migrate()
 	return p
 }

storage/storage_test.go

@@ -3,13 +3,16 @@ package storage
 import (
 	"elefant/models"
 	"fmt"
+	"log"
 	"log/slog"
 	"os"
 	"testing"
 	"time"
 
+	sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/ncruces"
 	_ "github.com/glebarez/go-sqlite"
 	"github.com/jmoiron/sqlx"
+	"github.com/ncruces/go-sqlite3"
 )
 
 func TestMemories(t *testing.T) {
@@ -160,3 +163,88 @@ func TestChatHistory(t *testing.T) {
 		t.Errorf("Expected 0 chats, got %d", len(chats))
 	}
 }
+
+func TestVecTable(t *testing.T) {
+	// healthcheck
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	stmt, _, err := db.Prepare(`SELECT sqlite_version(), vec_version()`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	stmt.Step()
+	log.Printf("sqlite_version=%s, vec_version=%s\n", stmt.ColumnText(0), stmt.ColumnText(1))
+	stmt.Close()
+	// migration
+	err = db.Exec("CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4], chat_name TEXT NOT NULL)")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// data prep and insert
+	items := map[int][]float32{
+		1: {0.1, 0.1, 0.1, 0.1},
+		2: {0.2, 0.2, 0.2, 0.2},
+		3: {0.3, 0.3, 0.3, 0.3},
+		4: {0.4, 0.4, 0.4, 0.4},
+		5: {0.5, 0.5, 0.5, 0.5},
+	}
+	q := []float32{0.28, 0.3, 0.3, 0.3}
+	stmt, _, err = db.Prepare("INSERT INTO vec_items(rowid, embedding, chat_name) VALUES (?, ?, ?)")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for id, values := range items {
+		v, err := sqlite_vec.SerializeFloat32(values)
+		if err != nil {
+			t.Fatal(err)
+		}
+		stmt.BindInt(1, id)
+		stmt.BindBlob(2, v)
+		stmt.BindText(3, "some_chat")
+		err = stmt.Exec()
+		if err != nil {
+			t.Fatal(err)
+		}
+		stmt.Reset()
+	}
+	stmt.Close()
+	// select | vec search
+	stmt, _, err = db.Prepare(`
+		SELECT
+			rowid,
+			distance,
+			embedding
+		FROM vec_items
+		WHERE embedding MATCH ?
+		ORDER BY distance
+		LIMIT 3
+	`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	query, err := sqlite_vec.SerializeFloat32(q)
+	if err != nil {
+		t.Fatal(err)
+	}
+	stmt.BindBlob(1, query)
+	for stmt.Step() {
+		rowid := stmt.ColumnInt64(0)
+		distance := stmt.ColumnFloat(1)
+		emb := stmt.ColumnRawText(2)
+		floats := decodeUnsafe(emb)
+		log.Printf("rowid=%d, distance=%f, floats=%v\n", rowid, distance, floats)
+	}
+	if err := stmt.Err(); err != nil {
+		t.Fatal(err)
+	}
+	err = stmt.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = db.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

storage/vector.go

@@ -0,0 +1,89 @@
+package storage
+
+import (
+	"elefant/models"
+	"fmt"
+	"log"
+	"unsafe"
+
+	sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/ncruces"
+)
+
+type VectorRepo interface {
+	WriteVector(*models.VectorRow) error
+	SearchClosest(q [5120]float32) (*models.VectorRow, error)
+}
+
+var vecTableName = "embeddings"
+
+func (p ProviderSQL) WriteVector(row *models.VectorRow) error {
+	stmt, _, err := p.s3Conn.Prepare(
+		fmt.Sprintf("INSERT INTO %s(embedding, slug, raw_text) VALUES (?, ?, ?)", vecTableName))
+	defer stmt.Close()
+	if err != nil {
+		p.logger.Error("failed to prep a stmt", "error", err)
+		return err
+	}
+	v, err := sqlite_vec.SerializeFloat32(row.Embeddings)
+	if err != nil {
+		p.logger.Error("failed to serialize vector",
+			"emb-len", len(row.Embeddings), "error", err)
+		return err
+	}
+	stmt.BindInt(1, int(row.ID))
+	stmt.BindBlob(2, v)
+	stmt.BindText(3, row.Slug)
+	stmt.BindText(4, row.RawText)
+	err = stmt.Exec()
+	if err != nil {
+		p.logger.Error("failed exec a stmt", "error", err)
+		return err
+	}
+	return nil
+}
+
+func decodeUnsafe(bs []byte) []float32 {
+	return unsafe.Slice((*float32)(unsafe.Pointer(&bs[0])), len(bs)/4)
+}
+
+func (p ProviderSQL) SearchClosest(q [5120]float32) (*models.VectorRow, error) {
+	stmt, _, err := p.s3Conn.Prepare(`
+		SELECT
+			id,
+			distance,
+			embedding,
+			slug,
+			raw_text
+		FROM vec_items
+		WHERE embedding MATCH ?
+		ORDER BY distance
+		LIMIT 4
+	`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	query, err := sqlite_vec.SerializeFloat32(q[:])
+	if err != nil {
+		log.Fatal(err)
+	}
+	stmt.BindBlob(1, query)
+	resp := make([]models.VectorRow, 4)
+	i := 0
+	for stmt.Step() {
+		resp[i].ID = uint32(stmt.ColumnInt64(0))
+		resp[i].Distance = float32(stmt.ColumnFloat(1))
+		emb := stmt.ColumnRawText(2)
+		resp[i].Embeddings = decodeUnsafe(emb)
+		resp[i].Slug = stmt.ColumnText(3)
+		resp[i].RawText = stmt.ColumnText(4)
+		i++
+	}
+	if err := stmt.Err(); err != nil {
+		log.Fatal(err)
+	}
+	err = stmt.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+	return nil, nil
+}