grail/gf-lt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into feat/agent-flow

Browse Source
This commit is contained in:
Grail Finder
2026-03-14 12:38:46 +03:00
parent f4fcb85570 4cfe2fe37f
commit 77c365959d
10 changed files with 162 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
storage/migrate.go
View File

@@ -23,9 +23,20 @@ func (p *ProviderSQL) Migrate() error {
p.logger.Error("Failed to read migrations directory;", "error", err)
return fmt.Errorf("failed to read migrations directory: %w", err)
}
// Check if FTS already has data - skip populate migration if so
var ftsCount int
_ = p.db.QueryRow("SELECT COUNT(*) FROM fts_embeddings").Scan(&ftsCount)
skipFTSMigration := ftsCount > 0
// Execute each .up.sql file
for _, file := range files {
if strings.HasSuffix(file.Name(), ".up.sql") {
// Skip FTS populate migration if already populated
if skipFTSMigration && strings.Contains(file.Name(), "004_populate_fts") {
p.logger.Debug("Skipping FTS migration - already populated", "file", file.Name())
continue
}
err := p.executeMigration(migrationsDir, file.Name())
if err != nil {
p.logger.Error("Failed to execute migration %s: %v", file.Name(), err)

30
storage/migrations/004_populate_fts.up.sql
View File

@@ -1,26 +1,4 @@
-- Populate FTS table with existing embeddings
DELETE FROM fts_embeddings;
INSERT INTO fts_embeddings (slug, raw_text, filename, embedding_size)
SELECT slug, raw_text, filename, 384 FROM embeddings_384;
INSERT INTO fts_embeddings (slug, raw_text, filename, embedding_size)
SELECT slug, raw_text, filename, 768 FROM embeddings_768;
INSERT INTO fts_embeddings (slug, raw_text, filename, embedding_size)
SELECT slug, raw_text, filename, 1024 FROM embeddings_1024;
INSERT INTO fts_embeddings (slug, raw_text, filename, embedding_size)
SELECT slug, raw_text, filename, 1536 FROM embeddings_1536;
INSERT INTO fts_embeddings (slug, raw_text, filename, embedding_size)
SELECT slug, raw_text, filename, 2048 FROM embeddings_2048;
INSERT INTO fts_embeddings (slug, raw_text, filename, embedding_size)
SELECT slug, raw_text, filename, 3072 FROM embeddings_3072;
INSERT INTO fts_embeddings (slug, raw_text, filename, embedding_size)
SELECT slug, raw_text, filename, 4096 FROM embeddings_4096;
INSERT INTO fts_embeddings (slug, raw_text, filename, embedding_size)
SELECT slug, raw_text, filename, 5120 FROM embeddings_5120;
-- Populate FTS table with existing embeddings (incremental - only inserts missing rows)
-- Only use 768 embeddings as that's what we use
INSERT OR IGNORE INTO fts_embeddings (slug, raw_text, filename, embedding_size)
SELECT slug, raw_text, filename, 768 FROM embeddings_768;

87
storage/migrations/005_drop_unused_embeddings.down.sql Normal file
View File

@@ -0,0 +1,87 @@
-- Recreate unused embedding tables (for rollback)
CREATE TABLE IF NOT EXISTS embeddings_384 (
id INTEGER PRIMARY KEY AUTOINCREMENT,
embeddings BLOB NOT NULL,
slug TEXT NOT NULL,
raw_text TEXT NOT NULL,
filename TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS embeddings_1024 (
id INTEGER PRIMARY KEY AUTOINCREMENT,
embeddings BLOB NOT NULL,
slug TEXT NOT NULL,
raw_text TEXT NOT NULL,
filename TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS embeddings_1536 (
id INTEGER PRIMARY KEY AUTOINCREMENT,
embeddings BLOB NOT NULL,
slug TEXT NOT NULL,
raw_text TEXT NOT NULL,
filename TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS embeddings_2048 (
id INTEGER PRIMARY KEY AUTOINCREMENT,
embeddings BLOB NOT NULL,
slug TEXT NOT NULL,
raw_text TEXT NOT NULL,
filename TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS embeddings_3072 (
id INTEGER PRIMARY KEY AUTOINCREMENT,
embeddings BLOB NOT NULL,
slug TEXT NOT NULL,
raw_text TEXT NOT NULL,
filename TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS embeddings_4096 (
id INTEGER PRIMARY KEY AUTOINCREMENT,
embeddings BLOB NOT NULL,
slug TEXT NOT NULL,
raw_text TEXT NOT NULL,
filename TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS embeddings_5120 (
id INTEGER PRIMARY KEY AUTOINCREMENT,
embeddings BLOB NOT NULL,
slug TEXT NOT NULL,
raw_text TEXT NOT NULL,
filename TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_embeddings_384_filename ON embeddings_384(filename);
CREATE INDEX IF NOT EXISTS idx_embeddings_1024_filename ON embeddings_1024(filename);
CREATE INDEX IF NOT EXISTS idx_embeddings_1536_filename ON embeddings_1536(filename);
CREATE INDEX IF NOT EXISTS idx_embeddings_2048_filename ON embeddings_2048(filename);
CREATE INDEX IF NOT EXISTS idx_embeddings_3072_filename ON embeddings_3072(filename);
CREATE INDEX IF NOT EXISTS idx_embeddings_4096_filename ON embeddings_4096(filename);
CREATE INDEX IF NOT EXISTS idx_embeddings_5120_filename ON embeddings_5120(filename);
CREATE INDEX IF NOT EXISTS idx_embeddings_384_slug ON embeddings_384(slug);
CREATE INDEX IF NOT EXISTS idx_embeddings_1024_slug ON embeddings_1024(slug);
CREATE INDEX IF NOT EXISTS idx_embeddings_1536_slug ON embeddings_1536(slug);
CREATE INDEX IF NOT EXISTS idx_embeddings_2048_slug ON embeddings_2048(slug);
CREATE INDEX IF NOT EXISTS idx_embeddings_3072_slug ON embeddings_3072(slug);
CREATE INDEX IF NOT EXISTS idx_embeddings_4096_slug ON embeddings_4096(slug);
CREATE INDEX IF NOT EXISTS idx_embeddings_5120_slug ON embeddings_5120(slug);
CREATE INDEX IF NOT EXISTS idx_embeddings_384_created_at ON embeddings_384(created_at);
CREATE INDEX IF NOT EXISTS idx_embeddings_1024_created_at ON embeddings_1024(created_at);
CREATE INDEX IF NOT EXISTS idx_embeddings_1536_created_at ON embeddings_1536(created_at);
CREATE INDEX IF NOT EXISTS idx_embeddings_2048_created_at ON embeddings_2048(created_at);
CREATE INDEX IF NOT EXISTS idx_embeddings_3072_created_at ON embeddings_3072(created_at);
CREATE INDEX IF NOT EXISTS idx_embeddings_4096_created_at ON embeddings_4096(created_at);
CREATE INDEX IF NOT EXISTS idx_embeddings_5120_created_at ON embeddings_5120(created_at);

32
storage/migrations/005_drop_unused_embeddings.up.sql Normal file
View File

@@ -0,0 +1,32 @@
-- Drop unused embedding tables (we only use 768)
DROP INDEX IF EXISTS idx_embeddings_384_filename;
DROP INDEX IF EXISTS idx_embeddings_1024_filename;
DROP INDEX IF EXISTS idx_embeddings_1536_filename;
DROP INDEX IF EXISTS idx_embeddings_2048_filename;
DROP INDEX IF EXISTS idx_embeddings_3072_filename;
DROP INDEX IF EXISTS idx_embeddings_4096_filename;
DROP INDEX IF EXISTS idx_embeddings_5120_filename;
DROP INDEX IF EXISTS idx_embeddings_384_slug;
DROP INDEX IF EXISTS idx_embeddings_1024_slug;
DROP INDEX IF EXISTS idx_embeddings_1536_slug;
DROP INDEX IF EXISTS idx_embeddings_2048_slug;
DROP INDEX IF EXISTS idx_embeddings_3072_slug;
DROP INDEX IF EXISTS idx_embeddings_4096_slug;
DROP INDEX IF EXISTS idx_embeddings_5120_slug;
DROP INDEX IF EXISTS idx_embeddings_384_created_at;
DROP INDEX IF EXISTS idx_embeddings_1024_created_at;
DROP INDEX IF EXISTS idx_embeddings_1536_created_at;
DROP INDEX IF EXISTS idx_embeddings_2048_created_at;
DROP INDEX IF EXISTS idx_embeddings_3072_created_at;
DROP INDEX IF EXISTS idx_embeddings_4096_created_at;
DROP INDEX IF EXISTS idx_embeddings_5120_created_at;
DROP TABLE IF EXISTS embeddings_384;
DROP TABLE IF EXISTS embeddings_1024;
DROP TABLE IF EXISTS embeddings_1536;
DROP TABLE IF EXISTS embeddings_2048;
DROP TABLE IF EXISTS embeddings_3072;
DROP TABLE IF EXISTS embeddings_4096;
DROP TABLE IF EXISTS embeddings_5120;

77
storage/vector.go
View File

@@ -48,22 +48,8 @@ func mathBitsToFloat32(b uint32) float32 {
func fetchTableName(emb []float32) (string, error) {
switch len(emb) {
case 384:
return "embeddings_384", nil
case 768:
return "embeddings_768", nil
case 1024:
return "embeddings_1024", nil
case 1536:
return "embeddings_1536", nil
case 2048:
return "embeddings_2048", nil
case 3072:
return "embeddings_3072", nil
case 4096:
return "embeddings_4096", nil
case 5120:
return "embeddings_5120", nil
default:
return "", fmt.Errorf("no table for the size of %d", len(emb))
}
@@ -170,62 +156,25 @@ func sqrt(f float32) float32 {
}
func (p ProviderSQL) ListFiles() ([]string, error) {
fileLists := make([][]string, 0)
// Query all supported tables and combine results
tableNames := []string{
"embeddings_384", "embeddings_768", "embeddings_1024", "embeddings_1536",
"embeddings_2048", "embeddings_3072", "embeddings_4096", "embeddings_5120",
query := "SELECT DISTINCT filename FROM embeddings_768"
rows, err := p.db.Query(query)
if err != nil {
return nil, err
}
for _, table := range tableNames {
query := "SELECT DISTINCT filename FROM " + table
rows, err := p.db.Query(query)
if err != nil {
// Continue if one table doesn't exist
defer rows.Close()
var allFiles []string
for rows.Next() {
var filename string
if err := rows.Scan(&filename); err != nil {
continue
}
var files []string
for rows.Next() {
var filename string
if err := rows.Scan(&filename); err != nil {
continue
}
files = append(files, filename)
}
rows.Close()
fileLists = append(fileLists, files)
}
// Combine and deduplicate
fileSet := make(map[string]bool)
var allFiles []string
for _, files := range fileLists {
for _, file := range files {
if !fileSet[file] {
fileSet[file] = true
allFiles = append(allFiles, file)
}
}
allFiles = append(allFiles, filename)
}
return allFiles, nil
}
func (p ProviderSQL) RemoveEmbByFileName(filename string) error {
var errors []string
tableNames := []string{
"embeddings_384", "embeddings_768", "embeddings_1024", "embeddings_1536",
"embeddings_2048", "embeddings_3072", "embeddings_4096", "embeddings_5120",
}
for _, table := range tableNames {
query := fmt.Sprintf("DELETE FROM %s WHERE filename = ?", table)
if _, err := p.db.Exec(query, filename); err != nil {
errors = append(errors, err.Error())
}
}
if len(errors) > 0 {
return fmt.Errorf("errors occurred: %v", errors)
}
return nil
query := "DELETE FROM embeddings_768 WHERE filename = ?"
_, err := p.db.Exec(query, filename)
return err
}