grail/gf-lt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enha (RAG): raw text as primary key in vector db

Browse Source
This commit is contained in:
Grail Finder
2025-01-07 11:11:44 +03:00
parent b822b3a161
commit 7bbedd93cf
9 changed files with 21 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
storage/migrations/002_add_vector.up.sql
View File

@@ -1,13 +1,11 @@
CREATE VIRTUAL TABLE IF NOT EXISTS embeddings USING vec0(
id INTEGER PRIMARY KEY AUTOINCREMENT,
embedding FLOAT[5120],
slug TEXT NOT NULL,
raw_text TEXT NOT NULL
raw_text TEXT PRIMARY KEY,
);
CREATE VIRTUAL TABLE IF NOT EXISTS embeddings_384 USING vec0(
id INTEGER PRIMARY KEY AUTOINCREMENT,
embedding FLOAT[384],
slug TEXT NOT NULL,
raw_text TEXT NOT NULL
raw_text TEXT PRIMARY KEY
);

11
storage/vector.go
View File

@@ -68,7 +68,6 @@ func (p ProviderSQL) WriteVector(row *models.VectorRow) error {
}
err = stmt.Exec()
if err != nil {
p.logger.Error("failed exec a stmt", "error", err)
return err
}
return nil
@@ -85,7 +84,6 @@ func (p ProviderSQL) SearchClosest(q []float32) ([]models.VectorRow, error) {
}
stmt, _, err := p.s3Conn.Prepare(
fmt.Sprintf(`SELECT
id,
distance,
embedding,
slug,
@@ -109,12 +107,11 @@ func (p ProviderSQL) SearchClosest(q []float32) ([]models.VectorRow, error) {
resp := []models.VectorRow{}
for stmt.Step() {
res := models.VectorRow{}
res.ID = uint32(stmt.ColumnInt64(0))
res.Distance = float32(stmt.ColumnFloat(1))
emb := stmt.ColumnRawText(2)
res.Distance = float32(stmt.ColumnFloat(0))
emb := stmt.ColumnRawText(1)
res.Embeddings = decodeUnsafe(emb)
res.Slug = stmt.ColumnText(3)
res.RawText = stmt.ColumnText(4)
res.Slug = stmt.ColumnText(2)
res.RawText = stmt.ColumnText(3)
resp = append(resp, res)
}
if err := stmt.Err(); err != nil {