Enha: onnx config vars

.gitignore

@@ -15,3 +15,4 @@ gflt
 chat_exports/*.json
 ragimport
 .env
+onnx/

Makefile

@@ -30,6 +30,9 @@ lint: ## Run linters. Use make install-linters first.
 lintall: lint
 	noblanks ./...
 
+fetch-onnx:
+	mkdir -p onnx/embedgemma && curl -o onnx/embedgemma/config.json -L https://huggingface.co/onnx-community/embeddinggemma-300m-ONNX/resolve/main/config.json && curl -o onnx/embedgemma/tokenizer.json -L https://huggingface.co/onnx-community/embeddinggemma-300m-ONNX/resolve/main/tokenizer.json && curl -o onnx/embedgemma/model_q4.onnx -L https://huggingface.co/onnx-community/embeddinggemma-300m-ONNX/resolve/main/onnx/model_q4.onnx && curl -o onnx/embedgemma/model_q4.onnx_data -L https://huggingface.co/onnx-community/embeddinggemma-300m-ONNX/resolve/main/onnx/model_q4.onnx_data?download=true
+
 # Whisper STT Setup (in batteries directory)
 setup-whisper: build-whisper download-whisper-model
 

bot.go

@@ -1393,12 +1393,13 @@ func updateModelLists() {
 		}
 	}
 	// if llama.cpp started after gf-lt?
-	localModelsMu.Lock()
+	ml, err := fetchLCPModelsWithLoadStatus()
-	LocalModels, err = fetchLCPModelsWithLoadStatus()
-	localModelsMu.Unlock()
 	if err != nil {
 		logger.Warn("failed to fetch llama.cpp models", "error", err)
 	}
+	localModelsMu.Lock()
+	LocalModels = ml
+	localModelsMu.Unlock()
 	// set already loaded model in llama.cpp
 	if strings.Contains(cfg.CurrentAPI, "localhost") || strings.Contains(cfg.CurrentAPI, "127.0.0.1") {
 		localModelsMu.Lock()

config.example.toml

@@ -13,6 +13,9 @@ OpenRouterChatAPI = "https://openrouter.ai/api/v1/chat/completions"
 # embeddings
 EmbedURL = "http://localhost:8082/v1/embeddings"
 HFToken = ""
+EmbedModelPath = "onnx/embedgemma/model_q4.onnx"
+EmbedTokenizerPath = "onnx/embedgemma/tokenizer.json"
+EmbedDims = 768
 #
 ShowSys = true
 LogFile = "log.txt"

config/config.go

@@ -36,6 +36,9 @@ type Config struct {
 	// embeddings
 	EmbedURL           string `toml:"EmbedURL"`
 	HFToken            string `toml:"HFToken"`
+	EmbedModelPath     string `toml:"EmbedModelPath"`
+	EmbedTokenizerPath string `toml:"EmbedTokenizerPath"`
+	EmbedDims          int    `toml:"EmbedDims"`
 	// rag settings
 	RAGEnabled   bool   `toml:"RAGEnabled"`
 	RAGDir       string `toml:"RAGDir"`

go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/GrailFinder/google-translate-tts v0.1.3
 	github.com/GrailFinder/searchagent v0.2.0
 	github.com/PuerkitoBio/goquery v1.11.0
-	github.com/deckarep/golang-set/v2 v2.8.0
 	github.com/gdamore/tcell/v2 v2.13.2
 	github.com/glebarez/go-sqlite v1.22.0
 	github.com/gopxl/beep/v2 v2.1.1
@@ -17,14 +16,18 @@ require (
 	github.com/neurosnap/sentences v1.1.2
 	github.com/playwright-community/playwright-go v0.5700.1
 	github.com/rivo/tview v0.42.0
+	github.com/sugarme/tokenizer v0.3.0
+	github.com/yalue/onnxruntime_go v1.27.0
 	github.com/yuin/goldmark v1.4.13
 )
 
 require (
 	github.com/andybalholm/cascadia v1.3.3 // indirect
+	github.com/deckarep/golang-set/v2 v2.8.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/ebitengine/oto/v3 v3.4.0 // indirect
 	github.com/ebitengine/purego v0.9.1 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/gdamore/encoding v1.0.1 // indirect
 	github.com/go-jose/go-jose/v3 v3.0.4 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
@@ -33,10 +36,14 @@ require (
 	github.com/hajimehoshi/oto/v2 v2.3.1 // indirect
 	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/schollz/progressbar/v2 v2.15.0 // indirect
+	github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c // indirect
 	golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 // indirect
 	golang.org/x/net v0.48.0 // indirect
 	golang.org/x/sys v0.39.0 // indirect

go.sum

@@ -21,6 +21,8 @@ github.com/ebitengine/oto/v3 v3.4.0 h1:br0PgASsEWaoWn38b2Goe7m1GKFYfNgnsjSd5Gg+/
 github.com/ebitengine/oto/v3 v3.4.0/go.mod h1:IOleLVD0m+CMak3mRVwsYY8vTctQgOM0iiL6S7Ar7eI=
 github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
 github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/gdamore/encoding v1.0.1 h1:YzKZckdBL6jVt2Gc+5p82qhrGiqMdG/eNs6Wy0u3Uhw=
 github.com/gdamore/encoding v1.0.1/go.mod h1:0Z0cMFinngz9kS1QfMjCP8TY7em3bZYeeklsSDPivEo=
 github.com/gdamore/tcell/v2 v2.13.2 h1:5j4srfF8ow3HICOv/61/sOhQtA25qxEB2XR3Q/Bhx2g=
@@ -61,10 +63,14 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
 github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/neurosnap/sentences v1.1.2 h1:iphYOzx/XckXeBiLIUBkPu2EKMJ+6jDbz/sLJZ7ZoUw=
 github.com/neurosnap/sentences v1.1.2/go.mod h1:/pwU4E9XNL21ygMIkOIllv/SMy2ujHwpf8GQPu1YPbQ=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/playwright-community/playwright-go v0.5700.1 h1:PNFb1byWqrTT720rEO0JL88C6Ju0EmUnR5deFLvtP/U=
@@ -77,10 +83,19 @@ github.com/rivo/tview v0.42.0 h1:b/ftp+RxtDsHSaynXTbJb+/n/BxDEi+W3UfF5jILK6c=
 github.com/rivo/tview v0.42.0/go.mod h1:cSfIYfhpSGCjp3r/ECJb+GKS7cGJnqV8vfjQPwoXyfY=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/schollz/progressbar/v2 v2.15.0 h1:dVzHQ8fHRmtPjD3K10jT3Qgn/+H+92jhPrhmxIJfDz8=
+github.com/schollz/progressbar/v2 v2.15.0/go.mod h1:UdPq3prGkfQ7MOzZKlDRpYKcFqEMczbD7YmbPgpzKMI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c h1:pwb4kNSHb4K89ymCaN+5lPH/MwnfSVg4rzGDh4d+iy4=
+github.com/sugarme/regexpset v0.0.0-20200920021344-4d4ec8eaf93c/go.mod h1:2gwkXLWbDGUQWeL3RtpCmcY4mzCtU13kb9UsAg9xMaw=
+github.com/sugarme/tokenizer v0.3.0 h1:FE8DYbNSz/kSbgEo9l/RjgYHkIJYEdskumitFQBE9FE=
+github.com/sugarme/tokenizer v0.3.0/go.mod h1:VJ+DLK5ZEZwzvODOWwY0cw+B1dabTd3nCB5HuFCItCc=
+github.com/yalue/onnxruntime_go v1.27.0 h1:c1YSgDNtpf0WGtxj3YeRIb8VC5LmM1J+Ve3uHdteC1U=
+github.com/yalue/onnxruntime_go v1.27.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
 github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

helpfuncs.go

@@ -521,7 +521,7 @@ func updateFlexLayout() {
 	if shellMode {
 		flex.AddItem(shellInput, 0, 10, false)
 	} else {
-		flex.AddItem(textArea, 0, 10, false)
+		flex.AddItem(bottomFlex, 0, 10, true)
 	}
 	if positionVisible {
 		flex.AddItem(statusLineWidget, 0, 2, false)

rag/embedder.go

@@ -9,6 +9,11 @@ import (
 	"gf-lt/models"
 	"log/slog"
 	"net/http"
+	"sync"
+
+	"github.com/sugarme/tokenizer"
+	"github.com/sugarme/tokenizer/pretrained"
+	"github.com/yalue/onnxruntime_go"
 )
 
 // Embedder defines the interface for embedding text
@@ -134,11 +139,160 @@ func (a *APIEmbedder) EmbedSlice(lines []string) ([][]float32, error) {
 	return embeddings, nil
 }
 
-// TODO: ONNXEmbedder implementation would go here
-// This would require:
 // 1. Loading ONNX models locally
 // 2. Using a Go ONNX runtime (like gorgonia/onnx or similar)
 // 3. Converting text to embeddings without external API calls
-//
+type ONNXEmbedder struct {
-// For now, we'll focus on the API implementation which is already working in the current system,
+	session   *onnxruntime_go.DynamicAdvancedSession
-// and can be extended later when we have ONNX runtime integration
+	tokenizer *tokenizer.Tokenizer
+	dims      int // embedding dimension (e.g., 768)
+	logger    *slog.Logger
+}
+
+var onnxInitOnce sync.Once
+
+func NewONNXEmbedder(modelPath, tokenizerPath string, dims int, logger *slog.Logger) (*ONNXEmbedder, error) {
+	// Initialize ONNX runtime environment once
+	onnxInitOnce.Do(func() {
+		onnxruntime_go.SetSharedLibraryPath("/usr/local/lib/libonnxruntime.so")
+		err := onnxruntime_go.InitializeEnvironment()
+		if err != nil {
+			logger.Error("failed to initialize ONNX runtime", "error", err)
+		}
+	})
+	// Load tokenizer using sugarme/tokenizer
+	tok, err := pretrained.FromFile(tokenizerPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load tokenizer: %w", err)
+	}
+	// Create ONNX session
+	session, err := onnxruntime_go.NewDynamicAdvancedSession(
+		modelPath, // onnx/embedgemma/model_q4.onnx
+		[]string{"input_ids", "attention_mask"},
+		[]string{"sentence_embedding"},
+		nil, // optional options
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create ONNX session: %w", err)
+	}
+	return &ONNXEmbedder{
+		session:   session,
+		tokenizer: tok,
+		dims:      dims,
+		logger:    logger,
+	}, nil
+}
+
+func (e *ONNXEmbedder) Embed(text string) ([]float32, error) {
+	// 1. Tokenize
+	encoding, err := e.tokenizer.EncodeSingle(text)
+	if err != nil {
+		return nil, fmt.Errorf("tokenization failed: %w", err)
+	}
+	// 2. Convert to int64 and create attention mask
+	ids := encoding.Ids
+	inputIDs := make([]int64, len(ids))
+	attentionMask := make([]int64, len(ids))
+	for i, id := range ids {
+		inputIDs[i] = int64(id)
+		attentionMask[i] = 1
+	}
+	// 3. Create input tensors (shape: [1, seq_len])
+	seqLen := int64(len(inputIDs))
+	inputIDsTensor, err := onnxruntime_go.NewTensor[int64](
+		onnxruntime_go.NewShape(1, seqLen),
+		inputIDs,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create input_ids tensor: %w", err)
+	}
+	defer func() { _ = inputIDsTensor.Destroy() }()
+	maskTensor, err := onnxruntime_go.NewTensor[int64](
+		onnxruntime_go.NewShape(1, seqLen),
+		attentionMask,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create attention_mask tensor: %w", err)
+	}
+	defer func() { _ = maskTensor.Destroy() }()
+	// 4. Create output tensor
+	outputTensor, err := onnxruntime_go.NewEmptyTensor[float32](
+		onnxruntime_go.NewShape(1, int64(e.dims)),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create output tensor: %w", err)
+	}
+	defer func() { _ = outputTensor.Destroy() }()
+	// 5. Run inference
+	err = e.session.Run(
+		[]onnxruntime_go.Value{inputIDsTensor, maskTensor},
+		[]onnxruntime_go.Value{outputTensor},
+	)
+	if err != nil {
+		return nil, fmt.Errorf("inference failed: %w", err)
+	}
+	// 6. Copy output data
+	outputData := outputTensor.GetData()
+	embedding := make([]float32, len(outputData))
+	copy(embedding, outputData)
+	return embedding, nil
+}
+
+func (e *ONNXEmbedder) EmbedSlice(texts []string) ([][]float32, error) {
+	encodings := make([]*tokenizer.Encoding, len(texts))
+	maxLen := 0
+	for i, txt := range texts {
+		enc, err := e.tokenizer.EncodeSingle(txt)
+		if err != nil {
+			return nil, err
+		}
+		encodings[i] = enc
+		if l := len(enc.Ids); l > maxLen {
+			maxLen = l
+		}
+	}
+	batchSize := len(texts)
+	inputIDs := make([]int64, batchSize*maxLen)
+	attentionMask := make([]int64, batchSize*maxLen)
+	for i, enc := range encodings {
+		ids := enc.Ids
+		offset := i * maxLen
+		for j, id := range ids {
+			inputIDs[offset+j] = int64(id)
+			attentionMask[offset+j] = 1
+		}
+		// Remaining positions are already zero (padding)
+	}
+	// Create tensors with shape [batchSize, maxLen]
+	inputTensor, _ := onnxruntime_go.NewTensor[int64](
+		onnxruntime_go.NewShape(int64(batchSize), int64(maxLen)),
+		inputIDs,
+	)
+	defer func() { _ = inputTensor.Destroy() }()
+	maskTensor, _ := onnxruntime_go.NewTensor[int64](
+		onnxruntime_go.NewShape(int64(batchSize), int64(maxLen)),
+		attentionMask,
+	)
+	defer func() { _ = maskTensor.Destroy() }()
+	outputTensor, _ := onnxruntime_go.NewEmptyTensor[float32](
+		onnxruntime_go.NewShape(int64(batchSize), int64(e.dims)),
+	)
+	defer func() { _ = outputTensor.Destroy() }()
+	err := e.session.Run(
+		[]onnxruntime_go.Value{inputTensor, maskTensor},
+		[]onnxruntime_go.Value{outputTensor},
+	)
+	if err != nil {
+		return nil, err
+	}
+	// Extract embeddings per batch item
+	data := outputTensor.GetData()
+	embeddings := make([][]float32, batchSize)
+	for i := 0; i < batchSize; i++ {
+		start := i * e.dims
+		emb := make([]float32, e.dims)
+		copy(emb, data[start:start+e.dims])
+		embeddings[i] = emb
+	}
+	return embeddings, nil
+}

rag/rag.go

@@ -34,8 +34,20 @@ type RAG struct {
 }
 
 func New(l *slog.Logger, s storage.FullRepo, cfg *config.Config) *RAG {
-	// Initialize with API embedder by default, could be configurable later
+	var embedder Embedder
-	embedder := NewAPIEmbedder(l, cfg)
+	if cfg.EmbedModelPath != "" && cfg.EmbedTokenizerPath != "" {
+		emb, err := NewONNXEmbedder(cfg.EmbedModelPath, cfg.EmbedTokenizerPath, cfg.EmbedDims, l)
+		if err != nil {
+			l.Error("failed to create ONNX embedder, falling back to API", "error", err)
+			embedder = NewAPIEmbedder(l, cfg)
+		} else {
+			embedder = emb
+			l.Info("using ONNX embedder", "model", cfg.EmbedModelPath, "dims", cfg.EmbedDims)
+		}
+	} else {
+		embedder = NewAPIEmbedder(l, cfg)
+		l.Info("using API embedder", "url", cfg.EmbedURL)
+	}
 	rag := &RAG{
 		logger:   l,
 		store:    s,
@@ -246,7 +258,7 @@ func (r *RAG) extractImportantPhrases(query string) string {
 				break
 			}
 		}
-		if isImportant || len(word) > 3 {
+		if isImportant || len(word) >= 3 {
 			important = append(important, word)
 		}
 	}

rag_issues.md

@@ -0,0 +1,104 @@
+# RAG Implementation Issues and Proposed Solutions
+
+## Overview
+The current RAG system fails to retrieve relevant information for specific queries, as demonstrated by the inability to find the "two she bears" reference in the KJV Bible (2 Kings 2:23-24). While the system retrieves documents containing the word "bear", it misses the actual verse, indicating fundamental flaws in chunking, query processing, retrieval, and answer synthesis. Below we dissect each problem and propose concrete solutions.
+
+---
+
+### Problem 1: Chunking Destroys Semantic Coherence
+- **Problem description**  
+  The current chunking splits text into sentences and groups them by a simple word count threshold (`RAGWordLimit`). This ignores document structure (chapters, headings) and can cut through narrative units, scattering related content across multiple chunks. For the Bible query, the story of Elisha and the bears likely spans multiple verses; splitting it prevents any single chunk from containing the full context, diluting the embedding signal and making retrieval difficult.
+
+- **Proposed solution**  
+  - **Structure-aware chunking**: Use the EPUB’s internal structure (chapters, sections) to create chunks that align with logical content units (e.g., by chapter or story).  
+  - **Overlap between chunks**: Add a configurable overlap (e.g., 10–20% of chunk size) to preserve continuity, ensuring key phrases like "two she bears" are not split across boundaries.  
+  - **Rich metadata**: Store book name, chapter, and verse numbers with each chunk to enable filtering and source attribution.  
+  - **Fallback to recursive splitting**: For documents without clear structure, use a recursive character text splitter with overlap (similar to LangChain’s `RecursiveCharacterTextSplitter`) to maintain semantic boundaries (paragraphs, sentences).
+
+---
+
+### Problem 2: Query Refinement Strips Critical Context
+- **Problem description**  
+  `RefineQuery` removes stop words and applies keyword-based filtering that discards semantically important modifiers. For "two she bears", the word "she" (a gender modifier) may be treated as a stop word, leaving "two bears". This loses the specificity of the query and causes the embedding to drift toward generic "bear" contexts. The rule-based approach cannot understand that "she bears" is a key phrase in the biblical story.
+
+- **Proposed solution**  
+  - **Entity-aware query preservation**: Use a lightweight NLP model (e.g., spaCy or a BERT-based NER tagger) to identify and retain key entities (quantities, animals, names) while only removing truly irrelevant stop words.  
+  - **Intelligent query rewriting**: Employ a small LLM (or a set of transformation rules) to generate query variations that reflect likely biblical phrasing, e.g., "two bears came out of the wood" or "Elisha and the bears".  
+  - **Contextual stop word removal**: Instead of a static list, use a POS tagger to keep adjectives, nouns, and verbs while removing only function words that don't carry meaning.  
+  - **Disable refinement for short queries**: If the query is already concise (like "two she bears"), skip aggressive filtering.
+
+---
+
+### Problem 3: Embedding Similarity Fails for Rare or Specific Phrases
+- **Problem description**  
+  Dense embeddings excel at capturing semantic similarity but can fail when the query contains rare phrases or when the relevant passage is embedded in a noisy chunk. The verse "there came forth two she bears out of the wood" shares only the word "bears" with the query, and its embedding may be pulled toward the average of surrounding verses. Consequently, the similarity score may be lower than that of other chunks containing the word "bear" in generic contexts.
+
+- **Proposed solution**  
+  - **Hybrid retrieval**: Combine dense embeddings with BM25 (keyword) search. BM25 excels at exact phrase matching and would likely retrieve the verse based on "two bears" even if the embedding is weak.  
+    - Use a library like [blevesearch](https://github.com/blevesearch/bleve) to index text alongside vectors.  
+    - Fuse results using Reciprocal Rank Fusion (RRF) or a weighted combination.  
+  - **Query expansion**: Add relevant terms to the query (e.g., "Elisha", "2 Kings") to improve embedding alignment.  
+  - **Fine-tuned embeddings**: Consider using an embedding model fine-tuned on domain-specific data (e.g., biblical texts) if this is a recurring use case.  
+
+---
+
+### Problem 4: Reranking Heuristics Are Insufficient
+- **Problem description**  
+  `RerankResults` boosts results based on simple keyword matching and file name heuristics. This coarse approach cannot reliably promote the correct verse over false positives. The adjustment `distance - score/100` is arbitrary and may not reflect true relevance.
+
+- **Proposed solution**  
+  - **Cross-encoder reranking**: After retrieving top candidates (e.g., top 20) with hybrid search, rerank them using a cross-encoder model that directly computes the relevance score between the query and each chunk.  
+    - Models like `cross-encoder/ms-marco-MiniLM-L-6-v2` are lightweight and can be run locally or via a microservice.  
+  - **Score normalization**: Use the cross-encoder scores to reorder results, discarding low-scoring ones.  
+  - **Contextual boosting**: If metadata (e.g., chapter/verse) is available, boost results that match the query’s expected location (if inferable).  
+
+---
+
+### Problem 5: Answer Synthesis Is Not Generative
+- **Problem description**  
+  `SynthesizeAnswer` embeds a prompt and attempts to retrieve a pre-stored answer, falling back to concatenating truncated chunks. This is fundamentally flawed: RAG requires an LLM to generate a coherent answer from retrieved context. In the Bible example, even if the correct verse were retrieved, the system would only output a snippet, not an answer explaining the reference.
+
+- **Proposed solution**  
+  - **Integrate an LLM for generation**: Use a local model (via Ollama, Llama.cpp) or a cloud API (OpenAI, etc.) to synthesize answers.  
+    - Construct a prompt that includes the retrieved chunks (with metadata) and the user query.  
+    - Instruct the model to answer based solely on the provided context and cite sources (e.g., "According to 2 Kings 2:24...").  
+  - **Implement a fallback**: If no relevant chunks are retrieved, return a message like "I couldn't find that information in your documents."  
+  - **Streaming support**: For better UX, stream the answer token-by-token.  
+
+---
+
+### Problem 6: Concurrency and Error Handling
+- **Problem description**  
+  The code uses a mutex only in `LoadRAG`, leaving other methods vulnerable to race conditions. The global status channel `LongJobStatusCh` may drop messages due to `select/default`, and errors are sometimes logged but not propagated. Ingestion is synchronous and slow.
+
+- **Proposed solution**  
+  - **Add context support**: Pass `context.Context` to all methods to allow cancellation and timeouts.  
+  - **Worker pools for embedding**: Parallelize batch embedding with a controlled number of workers to respect API rate limits and speed up ingestion.  
+  - **Retry logic**: Implement exponential backoff for transient API errors.  
+  - **Replace global channel**: Use a callback or an injectable status reporter to avoid dropping messages.  
+  - **Fine-grained locking**: Protect shared state (e.g., `storage`) with appropriate synchronization.  
+
+---
+
+### Problem 7: Lack of Monitoring and Evaluation
+- **Problem description**  
+  There are no metrics to track retrieval quality, latency, or user satisfaction. The failure case was discovered manually; without systematic evaluation, regressions will go unnoticed.
+
+- **Proposed solution**  
+  - **Log key metrics**: Record query, retrieved chunk IDs, scores, and latency for each search.  
+  - **User feedback**: Add a mechanism for users to rate answers (thumbs up/down) and use this data to improve retrieval.  
+  - **Offline evaluation**: Create a test set of queries and expected relevant chunks (e.g., the Bible example) to measure recall@k, MRR, etc., and run it after each change.  
+
+---
+
+## Summary
+Fixing the RAG pipeline requires a multi-pronged approach:
+1. **Structure-aware chunking** with metadata.
+2. **Hybrid retrieval** (dense + sparse).
+3. **Query understanding** via entity preservation and intelligent rewriting.
+4. **Cross-encoder reranking** for precision.
+5. **LLM-based answer generation**.
+6. **Robust concurrency and error handling**.
+7. **Monitoring and evaluation** to track improvements.
+
+Implementing these changes will transform the system from a brittle keyword matcher into a reliable knowledge assistant capable of handling nuanced queries like the "two she bears" reference.

sysprompts/cluedo.json

@@ -1,7 +0,0 @@
-{
-  "sys_prompt": "A game of cluedo. Players are {{user}}, {{char}}, {{char2}};\n\nrooms: hall, lounge, dinning room kitchen, ballroom, conservatory, billiard room, library, study;\nweapons: candlestick, dagger, lead pipe, revolver, rope, spanner;\npeople: miss Scarlett, colonel Mustard, mrs. White, reverend Green, mrs. Peacock, professor Plum;\n\nA murder happened in a mansion with 9 rooms. Victim is dr. Black.\nPlayers goal is to find out who commited a murder, in what room and with what weapon.\nWeapons, people and rooms not involved in murder are distributed between players (as cards) by tool agent.\nThe objective of the game is to deduce the details of the murder. There are six characters, six murder weapons, and nine rooms, leaving the players with 324 possibilities. As soon as a player enters a room, they may make a suggestion as to the details, naming a suspect, the room they are in, and the weapon. For example: \"I suspect Professor Plum, in the Dining Room, with the candlestick\".\nOnce a player makes a suggestion, the others are called upon to disprove it.\nBefore the player's move, tool agent will remind that players their cards. There are two types of moves: making a suggestion (suggestion_move) and disproving other player suggestion (evidence_move);\nIn this version player wins when the correct details are named in the suggestion_move.\n\n<example_game>\n{{user}}:\nlet's start a game of cluedo!\ntool: cards of {{char}} are 'LEAD PIPE', 'BALLROOM', 'CONSERVATORY', 'STUDY', 'Mrs. White'; suggestion_move;\n{{char}}:\n(putting miss Scarlet into the Hall with the Revolver) \"I suspect miss Scarlett, in the Hall, with the revolver.\"\ntool: cards of {{char2}} are 'SPANNER', 'DAGGER', 'Professor Plum', 'LIBRARY', 'Mrs. Peacock'; evidence_move;\n{{char2}}:\n\"No objections.\" (no cards matching the suspicion of {{char}})\ntool: cards of {{user}} are 'Colonel Mustard', 'Miss Scarlett', 'DINNING ROOM', 'CANDLESTICK', 'HALL'; evidence_move;\n{{user}}:\n\"I object. Miss Scarlett is innocent.\" (shows card with 'Miss Scarlett')\ntool: cards of {{char2}} are 'SPANNER', 'DAGGER', 'Professor Plum', 'LIBRARY', 'Mrs. Peacock'; suggestion_move;\n{{char2}}:\n*So it was not Miss Scarlett, good to know.*\n(moves Mrs. White to the Billiard Room) \"It might have been Mrs. White, in the Billiard Room, with the Revolver.\"\ntool: cards of {{user}} are 'Colonel Mustard', 'Miss Scarlett', 'DINNING ROOM', 'CANDLESTICK', 'HALL'; evidence_move;\n{{user}}:\n(no matching cards for the assumption of {{char2}}) \"Sounds possible to me.\"\ntool: cards of {{char}} are 'LEAD PIPE', 'BALLROOM', 'CONSERVATORY', 'STUDY', 'Mrs. White'; evidence_move;\n{{char}}:\n(shows Mrs. White card) \"No. Was not Mrs. White\"\ntool: cards of {{user}} are 'Colonel Mustard', 'Miss Scarlett', 'DINNING ROOM', 'CANDLESTICK', 'HALL'; suggestion_move;\n{{user}}:\n*So not Mrs. White...* (moves Reverend Green into the Billiard Room) \"I suspect Reverend Green, in the Billiard Room, with the Revolver.\"\ntool: Correct. It was Reverend Green in the Billiard Room, with the revolver. {{user}} wins.\n</example_game>",
-  "role": "CluedoPlayer",
-  "role2": "CluedoEnjoyer",
-  "filepath": "sysprompts/cluedo.json",
-  "first_msg": "Hey guys! Want to play cluedo?"
-}

tui.go

@@ -29,6 +29,8 @@ var (
 	statusLineWidget   *tview.TextView
 	helpView           *tview.TextView
 	flex               *tview.Flex
+	bottomFlex         *tview.Flex
+	notificationWidget *tview.TextView
 	imgView            *tview.Image
 	defaultImage       = "sysprompts/llama.png"
 	indexPickWindow    *tview.InputField
@@ -36,6 +38,7 @@ var (
 	roleEditWindow     *tview.InputField
 	shellInput         *tview.InputField
 	confirmModal       *tview.Modal
+	toastTimer         *time.Timer
 	confirmPageName    = "confirm"
 	fullscreenMode     bool
 	positionVisible    bool = true
@@ -137,8 +140,8 @@ func setShellMode(enabled bool) {
 	}()
 }
 
-// showToast displays a temporary message in the top‑right corner.
+// showToast displays a temporary notification in the bottom-right corner.
-// It auto‑hides after 3 seconds and disappears when clicked.
+// It auto-hides after 3 seconds.
 func showToast(title, message string) {
 	sanitize := func(s string, maxLen int) string {
 		sanitized := strings.Map(func(r rune) rune {
@@ -154,6 +157,11 @@ func showToast(title, message string) {
 	}
 	title = sanitize(title, 50)
 	message = sanitize(message, 197)
+	if toastTimer != nil {
+		toastTimer.Stop()
+	}
+	// show blocking notification to not mess up flex
+	if fullscreenMode {
 		notification := tview.NewTextView().
 			SetTextAlign(tview.AlignCenter).
 			SetDynamicColors(true).
@@ -176,14 +184,44 @@ func showToast(title, message string) {
 		// Generate a unique page name (e.g., using timestamp) to allow multiple toasts.
 		pageName := fmt.Sprintf("toast-%d", time.Now().UnixNano())
 		pages.AddPage(pageName, background, true, true)
-	// Auto‑dismiss after 3 seconds.
+		// Auto‑dismiss after 2 seconds, since blocking is more annoying
-	time.AfterFunc(3*time.Second, func() {
+		time.AfterFunc(2*time.Second, func() {
 			app.QueueUpdateDraw(func() {
 				if pages.HasPage(pageName) {
 					pages.RemovePage(pageName)
 				}
 			})
 		})
+		return
+	}
+	notificationWidget.SetTitle(title)
+	notificationWidget.SetText(fmt.Sprintf("[yellow]%s[-]", message))
+	go func() {
+		app.QueueUpdateDraw(func() {
+			flex.RemoveItem(bottomFlex)
+			flex.RemoveItem(statusLineWidget)
+			bottomFlex = tview.NewFlex().SetDirection(tview.FlexColumn).
+				AddItem(textArea, 0, 1, true).
+				AddItem(notificationWidget, 40, 1, false)
+			flex.AddItem(bottomFlex, 0, 10, true)
+			if positionVisible {
+				flex.AddItem(statusLineWidget, 0, 2, false)
+			}
+		})
+	}()
+	toastTimer = time.AfterFunc(3*time.Second, func() {
+		app.QueueUpdateDraw(func() {
+			flex.RemoveItem(bottomFlex)
+			flex.RemoveItem(statusLineWidget)
+			bottomFlex = tview.NewFlex().SetDirection(tview.FlexColumn).
+				AddItem(textArea, 0, 1, true).
+				AddItem(notificationWidget, 0, 0, false)
+			flex.AddItem(bottomFlex, 0, 10, true)
+			if positionVisible {
+				flex.AddItem(statusLineWidget, 0, 2, false)
+			}
+		})
+	})
 }
 
 func init() {
@@ -235,7 +273,7 @@ func init() {
 			shellHistoryPos = -1
 		}
 		// Handle Tab key for @ file completion
-		if event.Key() == tcell.KeyTab {
+		if event.Key() == tcell.KeyTab && shellMode {
 			currentText := shellInput.GetText()
 			atIndex := strings.LastIndex(currentText, "@")
 			if atIndex >= 0 {
@@ -286,12 +324,26 @@ func init() {
 		SetDynamicColors(true).
 		SetRegions(true).
 		SetChangedFunc(func() {
+			// INFO:
+			// https://github.com/rivo/tview/wiki/Concurrency#event-handlers
+			// although already called by default per tview specs
+			// calling it explicitly makes text streaming to look more smooth
 			app.Draw()
 		})
+	notificationWidget = tview.NewTextView().
+		SetTextAlign(tview.AlignCenter).
+		SetDynamicColors(true).
+		SetRegions(true).
+		SetChangedFunc(func() {
+		})
+	notificationWidget.SetBorder(true).SetTitle("notification")
+	bottomFlex = tview.NewFlex().SetDirection(tview.FlexColumn).
+		AddItem(textArea, 0, 1, true).
+		AddItem(notificationWidget, 0, 0, false)
 	//
 	flex = tview.NewFlex().SetDirection(tview.FlexRow).
 		AddItem(textView, 0, 40, false).
-		AddItem(textArea, 0, 10, true) // Restore original height
+		AddItem(bottomFlex, 0, 10, true)
 	if positionVisible {
 		flex.AddItem(statusLineWidget, 0, 2, false)
 	}
@@ -360,10 +412,14 @@ func init() {
 	// 	y += h / 2
 	// 	return x, y, w, h
 	// })
+	notificationWidget.SetDrawFunc(func(screen tcell.Screen, x, y, w, h int) (int, int, int, int) {
+		y += h / 2
+		return x, y, w, h
+	})
 	// Initially set up flex without search bar
 	flex = tview.NewFlex().SetDirection(tview.FlexRow).
 		AddItem(textView, 0, 40, false).
-		AddItem(textArea, 0, 10, true) // Restore original height
+		AddItem(bottomFlex, 0, 10, true)
 	if positionVisible {
 		flex.AddItem(statusLineWidget, 0, 2, false)
 	}
@@ -1095,7 +1151,7 @@ func init() {
 			chatRoundChan <- &models.ChatRoundReq{Role: persona, UserMsg: msgText}
 			return nil
 		}
-		if event.Key() == tcell.KeyTab {
+		if event.Key() == tcell.KeyTab && !shellMode {
 			currentF := app.GetFocus()
 			if currentF == textArea {
 				currentText := textArea.GetText()