Enha: close rag on success

move cleaning image attachment to the end of chatRound
fmt cleanup

Makefile

@@ -1,4 +1,4 @@
-.PHONY: setconfig run lint setup-whisper build-whisper download-whisper-model docker-up docker-down docker-logs noextra-run
+.PHONY: setconfig run lint setup-whisper build-whisper download-whisper-model docker-up docker-down docker-logs noextra-run installdelve checkdelve
 
 run: setconfig
 	go build -tags extra -o gf-lt && ./gf-lt
@@ -15,6 +15,12 @@ noextra-run: setconfig
 setconfig:
 	find config.toml &>/dev/null || cp config.example.toml config.toml
 
+installdelve:
+	go install github.com/go-delve/delve/cmd/dlv@latest
+
+checkdelve:
+	which dlv &>/dev/null || installdelve
+
 lint: ## Run linters. Use make install-linters first.
 	golangci-lint run -c .golangci.yml ./...
 

bot.go

@@ -411,14 +411,21 @@ func fetchLCPModelsWithLoadStatus() ([]string, error) {
 		return nil, err
 	}
 	result := make([]string, 0, len(models.Data))
-	for _, m := range models.Data {
+	li := 0 // loaded index
+	for i, m := range models.Data {
 		modelName := m.ID
 		if m.Status.Value == "loaded" {
 			modelName = "(loaded) " + modelName
+			li = i
 		}
 		result = append(result, modelName)
 	}
-	return result, nil
+	if li == 0 {
+		return result, nil // no loaded models
+	}
+	loadedModel := result[li]
+	result = append(result[:li], result[li+1:]...)
+	return slices.Concat([]string{loadedModel}, result), nil
 }
 
 // fetchLCPModelsWithStatus returns the full LCPModels struct including status information.
@@ -569,7 +576,6 @@ func sendMsgToLLM(body io.Reader) {
 		streamDone <- true
 		return
 	}
-
 	// Check if the initial response is an error before starting to stream
 	if resp.StatusCode >= 400 {
 		// Read the response body to get detailed error information
@@ -584,7 +590,6 @@ func sendMsgToLLM(body io.Reader) {
 			streamDone <- true
 			return
 		}
-
 		// Parse the error response for detailed information
 		detailedError := extractDetailedErrorFromBytes(bodyBytes, resp.StatusCode)
 		logger.Error("API returned error status", "status_code", resp.StatusCode, "detailed_error", detailedError)
@@ -710,7 +715,6 @@ func sendMsgToLLM(body io.Reader) {
 				tokenCount++
 			}
 		}
-
 		// When we get content and have been streaming reasoning, close the thinking block
 		if chunk.Chunk != "" && hasReasoning && !reasoningSent {
 			// Close the thinking block before sending actual content
@@ -718,7 +722,6 @@ func sendMsgToLLM(body io.Reader) {
 			tokenCount++
 			reasoningSent = true
 		}
-
 		// bot sends way too many \n
 		answerText = strings.ReplaceAll(chunk.Chunk, "\n\n", "\n")
 		// Accumulate text to check for stop strings that might span across chunks
@@ -764,12 +767,10 @@ func chatRagUse(qText string) (string, error) {
 		questions[i] = q.Text
 		logger.Debug("RAG question extracted", "index", i, "question", q.Text)
 	}
-
 	if len(questions) == 0 {
 		logger.Warn("No questions extracted from query text", "query", qText)
 		return "No related results from RAG vector storage.", nil
 	}
-
 	respVecs := []models.VectorRow{}
 	for i, q := range questions {
 		logger.Debug("Processing RAG question", "index", i, "question", q)
@@ -779,7 +780,6 @@ func chatRagUse(qText string) (string, error) {
 			continue
 		}
 		logger.Debug("Got embeddings for question", "index", i, "question_len", len(q), "embedding_len", len(emb))
-
 		// Create EmbeddingResp struct for the search
 		embeddingResp := &models.EmbeddingResp{
 			Embedding: emb,
@@ -793,7 +793,6 @@ func chatRagUse(qText string) (string, error) {
 		logger.Debug("RAG search returned vectors", "index", i, "question", q, "vector_count", len(vecs))
 		respVecs = append(respVecs, vecs...)
 	}
-
 	// get raw text
 	resps := []string{}
 	logger.Debug("RAG query final results", "total_vecs_found", len(respVecs))
@@ -801,12 +800,10 @@ func chatRagUse(qText string) (string, error) {
 		resps = append(resps, rv.RawText)
 		logger.Debug("RAG result", "slug", rv.Slug, "filename", rv.FileName, "raw_text_len", len(rv.RawText))
 	}
-
 	if len(resps) == 0 {
 		logger.Info("No RAG results found for query", "original_query", qText, "question_count", len(questions))
 		return "No related results from RAG vector storage.", nil
 	}
-
 	result := strings.Join(resps, "\n")
 	logger.Debug("RAG query completed", "result_len", len(result), "response_count", len(resps))
 	return result, nil
@@ -836,7 +833,10 @@ func chatRound(r *models.ChatRoundReq) error {
 	if cfg.WriteNextMsgAsCompletionAgent != "" {
 		botPersona = cfg.WriteNextMsgAsCompletionAgent
 	}
-	defer func() { botRespMode = false }()
+	defer func() {
+		botRespMode = false
+		ClearImageAttachment()
+	}()
 	// check that there is a model set to use if is not local
 	choseChunkParser()
 	reader, err := chunkParser.FormMsg(r.UserMsg, r.Role, r.Resume)
@@ -855,13 +855,14 @@ func chatRound(r *models.ChatRoundReq) error {
 		chatBody.Messages = append(chatBody.Messages, models.RoleMsg{
 			Role: botPersona, Content: "",
 		})
-		fmt.Fprintf(textView, "\n[-:-:b](%d) ", msgIdx)
-		fmt.Fprint(textView, roleToIcon(botPersona))
-		fmt.Fprint(textView, "[-:-:-]\n")
-		if cfg.ThinkUse && !strings.Contains(cfg.CurrentAPI, "v1") {
-			// fmt.Fprint(textView, "<think>")
-			chunkChan <- "<think>"
+		nl := "\n\n"
+		prevText := textView.GetText(true)
+		if strings.HasSuffix(prevText, nl) {
+			nl = ""
+		} else if strings.HasSuffix(prevText, "\n") {
+			nl = "\n"
 		}
+		fmt.Fprintf(textView, "%s[-:-:b](%d) %s[-:-:-]\n", nl, msgIdx, roleToIcon(botPersona))
 	} else {
 		msgIdx = len(chatBody.Messages) - 1
 	}
@@ -1198,6 +1199,8 @@ func findCall(msg, toolCall string) bool {
 		chatRoundChan <- crr
 		return true
 	}
+	// Show tool call progress indicator before execution
+	fmt.Fprintf(textView, "\n[yellow::i][tool: %s...][-:-:-]", fc.Name)
 	resp := callToolWithAgent(fc.Name, fc.Args)
 	toolMsg := string(resp) // Remove the "tool response: " prefix and %+v formatting
 	logger.Info("llm used a tool call", "tool_name", fc.Name, "too_args", fc.Args, "id", fc.ID, "tool_resp", toolMsg)
@@ -1237,7 +1240,6 @@ func chatToTextSlice(messages []models.RoleMsg, showSys bool) []string {
 func chatToText(messages []models.RoleMsg, showSys bool) string {
 	s := chatToTextSlice(messages, showSys)
 	text := strings.Join(s, "\n")
-
 	// Collapse thinking blocks if enabled
 	if thinkingCollapsed {
 		text = thinkRE.ReplaceAllStringFunc(text, func(match string) string {
@@ -1261,7 +1263,6 @@ func chatToText(messages []models.RoleMsg, showSys bool) string {
 			}
 		}
 	}
-
 	return text
 }
 

config.example.toml

@@ -12,7 +12,7 @@ OpenRouterChatAPI = "https://openrouter.ai/api/v1/chat/completions"
 # OpenRouterToken = ""
 # embeddings
 EmbedURL = "http://localhost:8082/v1/embeddings"
-HFToken = false
+HFToken = ""
 #
 ShowSys = true
 LogFile = "log.txt"

config/config.go

@@ -18,7 +18,6 @@ type Config struct {
 	UserRole                      string `toml:"UserRole"`
 	ToolRole                      string `toml:"ToolRole"`
 	ToolUse                       bool   `toml:"ToolUse"`
-	ThinkUse                      bool   `toml:"ThinkUse"`
 	StripThinkingFromAPI          bool   `toml:"StripThinkingFromAPI"`
 	ReasoningEffort               string `toml:"ReasoningEffort"`
 	AssistantRole                 string `toml:"AssistantRole"`
@@ -125,6 +124,9 @@ func LoadConfig(fn string) (*Config, error) {
 	if config.CompletionAPI != "" {
 		config.ApiLinks = append(config.ApiLinks, config.CompletionAPI)
 	}
+	if config.RAGDir == "" {
+		config.RAGDir = "ragimport"
+	}
 	// if any value is empty fill with default
 	return config, nil
 }

docs/config.md

@@ -165,9 +165,6 @@ Those could be switched in program, but also bould be setup in config.
 #### ToolUse
 - Enable or disable explanation of tools to llm, so it could use them.
 
-#### ThinkUse
-- Enable or disable insertion of  JsonSerializerToken at the beggining of llm resp.
-
 ### StripThinkingFromAPI (`true`)
 - Strip thinking blocks from messages before sending to LLM. Keeps them in chat history for local viewing but reduces token usage in API calls.
 

llm.go

@@ -184,9 +184,6 @@ func (lcp LCPCompletion) FormMsg(msg, role string, resume bool) (io.Reader, erro
 		botMsgStart := "\n" + botPersona + ":\n"
 		prompt += botMsgStart
 	}
-	if cfg.ThinkUse && !cfg.ToolUse {
-		prompt += "<think>"
-	}
 	logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
 		"msg", msg, "resume", resume, "prompt", prompt, "multimodal_data_count", len(multimodalData))
 	payload := models.NewLCPReq(prompt, chatBody.Model, multimodalData,
@@ -423,9 +420,6 @@ func (ds DeepSeekerCompletion) FormMsg(msg, role string, resume bool) (io.Reader
 		botMsgStart := "\n" + botPersona + ":\n"
 		prompt += botMsgStart
 	}
-	if cfg.ThinkUse && !cfg.ToolUse {
-		prompt += "<think>"
-	}
 	logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
 		"msg", msg, "resume", resume, "prompt", prompt)
 	payload := models.NewDSCompletionReq(prompt, chatBody.Model,
@@ -589,9 +583,6 @@ func (or OpenRouterCompletion) FormMsg(msg, role string, resume bool) (io.Reader
 		botMsgStart := "\n" + botPersona + ":\n"
 		prompt += botMsgStart
 	}
-	if cfg.ThinkUse && !cfg.ToolUse {
-		prompt += "<think>"
-	}
 	stopSlice := chatBody.MakeStopSliceExcluding("", listChatRoles())
 	logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
 		"msg", msg, "resume", resume, "prompt", prompt, "stop_strings", stopSlice)

models/models.go

@@ -241,8 +241,7 @@ func (m *RoleMsg) ToText(i int) string {
 	}
 	finalContent.WriteString(contentStr)
 	if m.Stats != nil {
-		finalContent.WriteString(fmt.Sprintf("\n[gray::i][%d tok, %.1fs, %.1f t/s][-:-:-]",
-			m.Stats.Tokens, m.Stats.Duration, m.Stats.TokensPerSec))
+		fmt.Fprintf(&finalContent, "\n[gray::i][%d tok, %.1fs, %.1f t/s][-:-:-]", m.Stats.Tokens, m.Stats.Duration, m.Stats.TokensPerSec)
 	}
 	textMsg := fmt.Sprintf("[-:-:b]%s[-:-:-]\n%s\n", icon, finalContent.String())
 	return strings.ReplaceAll(textMsg, "\n\n", "\n")

popups.go

@@ -51,7 +51,7 @@ func showModelSelectionPopup() {
 	// Find the current model index to set as selected
 	currentModelIndex := -1
 	for i, model := range modelList {
-		if model == chatBody.Model {
+		if strings.TrimPrefix(model, "(loaded) ") == chatBody.Model {
 			currentModelIndex = i
 		}
 		modelListWidget.AddItem(model, "", 0, nil)
@@ -65,16 +65,19 @@ func showModelSelectionPopup() {
 		chatBody.Model = modelName
 		cfg.CurrentModel = chatBody.Model
 		pages.RemovePage("modelSelectionPopup")
+		app.SetFocus(textArea)
 		updateCachedModelColor()
 		updateStatusLine()
 	})
 	modelListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("modelSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("modelSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -160,6 +163,7 @@ func showAPILinkSelectionPopup() {
 			cfg.CurrentModel = chatBody.Model
 		}
 		pages.RemovePage("apiLinkSelectionPopup")
+		app.SetFocus(textArea)
 		choseChunkParser()
 		updateCachedModelColor()
 		updateStatusLine()
@@ -167,10 +171,12 @@ func showAPILinkSelectionPopup() {
 	apiListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("apiLinkSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("apiLinkSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -230,6 +236,7 @@ func showUserRoleSelectionPopup() {
 		textView.SetText(chatToText(filtered, cfg.ShowSys))
 		// Remove the popup page
 		pages.RemovePage("userRoleSelectionPopup")
+		app.SetFocus(textArea)
 		// Update the status line to reflect the change
 		updateStatusLine()
 		colorText()
@@ -237,10 +244,12 @@ func showUserRoleSelectionPopup() {
 	roleListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("userRoleSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("userRoleSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -303,16 +312,19 @@ func showBotRoleSelectionPopup() {
 		cfg.WriteNextMsgAsCompletionAgent = mainText
 		// Remove the popup page
 		pages.RemovePage("botRoleSelectionPopup")
+		app.SetFocus(textArea)
 		// Update the status line to reflect the change
 		updateStatusLine()
 	})
 	roleListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("botRoleSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("botRoleSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -364,14 +376,17 @@ func showFileCompletionPopup(filter string) {
 			textArea.SetText(before+mainText, true)
 		}
 		pages.RemovePage("fileCompletionPopup")
+		app.SetFocus(textArea)
 	})
 	widget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("fileCompletionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("fileCompletionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -484,14 +499,17 @@ func showColorschemeSelectionPopup() {
 		}
 		// Remove the popup page
 		pages.RemovePage("colorschemeSelectionPopup")
+		app.SetFocus(textArea)
 	})
 	schemeListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("colorschemeSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("colorschemeSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event

props_table.go

@@ -115,9 +115,6 @@ func makePropsTable(props map[string]float32) *tview.Table {
 		row++
 	}
 	// Add checkboxes
-	addCheckboxRow("Insert <think> tag (/completion only)", cfg.ThinkUse, func(checked bool) {
-		cfg.ThinkUse = checked
-	})
 	addCheckboxRow("RAG use", cfg.RAGEnabled, func(checked bool) {
 		cfg.RAGEnabled = checked
 	})

rag/rag.go

@@ -23,13 +23,13 @@ var (
 	ErrRAGStatus        = "some error occurred; failed to transfer data to vector db"
 )
 
-
 type RAG struct {
 	logger   *slog.Logger
 	store    storage.FullRepo
 	cfg      *config.Config
 	embedder Embedder
 	storage  *VectorStorage
+	mu       sync.Mutex
 }
 
 func New(l *slog.Logger, s storage.FullRepo, cfg *config.Config) *RAG {
@@ -54,6 +54,8 @@ func wordCounter(sentence string) int {
 }
 
 func (r *RAG) LoadRAG(fpath string) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
 	data, err := os.ReadFile(fpath)
 	if err != nil {
 		return err
@@ -63,9 +65,7 @@ func (r *RAG) LoadRAG(fpath string) error {
 	case LongJobStatusCh <- LoadedFileRAGStatus:
 	default:
 		r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", LoadedFileRAGStatus)
-		// Channel is full or closed, ignore the message to prevent panic
 	}
-
 	fileText := string(data)
 	tokenizer, err := english.NewSentenceTokenizer(nil)
 	if err != nil {
@@ -76,19 +76,16 @@ func (r *RAG) LoadRAG(fpath string) error {
 	for i, s := range sentences {
 		sents[i] = s.Text
 	}
-
 	// Group sentences into paragraphs based on word limit
 	paragraphs := []string{}
 	par := strings.Builder{}
 	for i := 0; i < len(sents); i++ {
-		// Only add sentences that aren't empty
 		if strings.TrimSpace(sents[i]) != "" {
 			if par.Len() > 0 {
-				par.WriteString(" ") // Add space between sentences
+				par.WriteString(" ")
 			}
 			par.WriteString(sents[i])
 		}
-
 		if wordCounter(par.String()) > int(r.cfg.RAGWordLimit) {
 			paragraph := strings.TrimSpace(par.String())
 			if paragraph != "" {
@@ -97,7 +94,6 @@ func (r *RAG) LoadRAG(fpath string) error {
 			par.Reset()
 		}
 	}
-
 	// Handle any remaining content in the paragraph buffer
 	if par.Len() > 0 {
 		paragraph := strings.TrimSpace(par.String())
@@ -105,215 +101,82 @@ func (r *RAG) LoadRAG(fpath string) error {
 			paragraphs = append(paragraphs, paragraph)
 		}
 	}
-
 	// Adjust batch size if needed
 	if len(paragraphs) < r.cfg.RAGBatchSize && len(paragraphs) > 0 {
 		r.cfg.RAGBatchSize = len(paragraphs)
 	}
-
 	if len(paragraphs) == 0 {
 		return errors.New("no valid paragraphs found in file")
 	}
-
-	var (
-		maxChSize = 100
-		left      = 0
-		right     = r.cfg.RAGBatchSize
-		batchCh   = make(chan map[int][]string, maxChSize)
-		vectorCh  = make(chan []models.VectorRow, maxChSize)
-		errCh     = make(chan error, 1)
-		wg        = new(sync.WaitGroup)
-		lock      = new(sync.Mutex)
-	)
-
-	defer close(errCh)
-	defer close(batchCh)
-
-	// Fill input channel with batches
-	ctn := 0
-	totalParagraphs := len(paragraphs)
-	for {
-		if right > totalParagraphs {
-			batchCh <- map[int][]string{left: paragraphs[left:]}
-			break
+	// Process paragraphs in batches synchronously
+	batchCount := 0
+	for i := 0; i < len(paragraphs); i += r.cfg.RAGBatchSize {
+		end := i + r.cfg.RAGBatchSize
+		if end > len(paragraphs) {
+			end = len(paragraphs)
 		}
-		batchCh <- map[int][]string{left: paragraphs[left:right]}
-		left, right = right, right+r.cfg.RAGBatchSize
-		ctn++
-	}
-
-	finishedBatchesMsg := fmt.Sprintf("finished batching batches#: %d; paragraphs: %d; sentences: %d\n", ctn+1, len(paragraphs), len(sents))
-	r.logger.Debug(finishedBatchesMsg)
-	select {
-	case LongJobStatusCh <- finishedBatchesMsg:
-	default:
-		r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", finishedBatchesMsg)
-		// Channel is full or closed, ignore the message to prevent panic
-	}
-
-	// Start worker goroutines with WaitGroup
-	wg.Add(int(r.cfg.RAGWorkers))
-	for w := 0; w < int(r.cfg.RAGWorkers); w++ {
-		go func(workerID int) {
-			defer wg.Done()
-			r.batchToVectorAsync(lock, workerID, batchCh, vectorCh, errCh, path.Base(fpath))
-		}(w)
-	}
-
-	// Use a goroutine to close the batchCh when all batches are sent
-	go func() {
-		wg.Wait()
-		close(vectorCh) // Close vectorCh when all workers are done
-	}()
-
-	// Check for errors from workers
-	// Use a non-blocking check for errors
-	select {
-	case err := <-errCh:
+		batch := paragraphs[i:end]
+		batchCount++
+		// Filter empty paragraphs
+		nonEmptyBatch := make([]string, 0, len(batch))
+		for _, p := range batch {
+			if strings.TrimSpace(p) != "" {
+				nonEmptyBatch = append(nonEmptyBatch, strings.TrimSpace(p))
+			}
+		}
+		if len(nonEmptyBatch) == 0 {
+			continue
+		}
+		// Embed the batch
+		embeddings, err := r.embedder.EmbedSlice(nonEmptyBatch)
 		if err != nil {
-			r.logger.Error("error during RAG processing", "error", err)
+			r.logger.Error("failed to embed batch", "error", err, "batch", batchCount)
+			select {
+			case LongJobStatusCh <- ErrRAGStatus:
+			default:
+				r.logger.Warn("LongJobStatusCh channel full, dropping message")
+			}
+			return fmt.Errorf("failed to embed batch %d: %w", batchCount, err)
+		}
+		if len(embeddings) != len(nonEmptyBatch) {
+			err := errors.New("embedding count mismatch")
+			r.logger.Error("embedding mismatch", "expected", len(nonEmptyBatch), "got", len(embeddings))
 			return err
 		}
-	default:
-		// No immediate error, continue
-	}
-
-	// Write vectors to storage - this will block until vectorCh is closed
-	return r.writeVectors(vectorCh)
-}
-
-func (r *RAG) writeVectors(vectorCh chan []models.VectorRow) error {
-	for {
-		for batch := range vectorCh {
-			for _, vector := range batch {
-				if err := r.storage.WriteVector(&vector); err != nil {
-					r.logger.Error("failed to write vector to DB", "error", err, "slug", vector.Slug)
-					select {
-					case LongJobStatusCh <- ErrRAGStatus:
-					default:
-						r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", ErrRAGStatus)
-						// Channel is full or closed, ignore the message to prevent panic
-					}
-					return err // Stop the entire RAG operation on DB error
-				}
+		// Write vectors to storage
+		filename := path.Base(fpath)
+		for j, text := range nonEmptyBatch {
+			vector := models.VectorRow{
+				Embeddings: embeddings[j],
+				RawText:    text,
+				Slug:       fmt.Sprintf("%s_%d_%d", filename, batchCount, j),
+				FileName:   filename,
 			}
-			r.logger.Debug("wrote batch to db", "size", len(batch), "vector_chan_len", len(vectorCh))
-			if len(vectorCh) == 0 {
-				r.logger.Debug("finished writing vectors")
+			if err := r.storage.WriteVector(&vector); err != nil {
+				r.logger.Error("failed to write vector to DB", "error", err, "slug", vector.Slug)
 				select {
-				case LongJobStatusCh <- FinishedRAGStatus:
+				case LongJobStatusCh <- ErrRAGStatus:
 				default:
-					r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", FinishedRAGStatus)
-					// Channel is full or closed, ignore the message to prevent panic
+					r.logger.Warn("LongJobStatusCh channel full, dropping message")
 				}
-				return nil
+				return fmt.Errorf("failed to write vector: %w", err)
 			}
 		}
-	}
-}
-
-func (r *RAG) batchToVectorAsync(lock *sync.Mutex, id int, inputCh <-chan map[int][]string,
-	vectorCh chan<- []models.VectorRow, errCh chan error, filename string) {
-	var err error
-
-	defer func() {
-		// For errCh, make sure we only send if there's actually an error and the channel can accept it
-		if err != nil {
-			select {
-			case errCh <- err:
-			default:
-				// errCh might be full or closed, log but don't panic
-				r.logger.Warn("errCh channel full or closed, skipping error propagation", "worker", id, "error", err)
-			}
-		}
-	}()
-
-	for {
-		lock.Lock()
-		if len(inputCh) == 0 {
-			lock.Unlock()
-			return
-		}
-
-		select {
-		case linesMap := <-inputCh:
-			for leftI, lines := range linesMap {
-				if err := r.fetchEmb(lines, errCh, vectorCh, fmt.Sprintf("%s_%d", filename, leftI), filename); err != nil {
-					r.logger.Error("error fetching embeddings", "error", err, "worker", id)
-					lock.Unlock()
-					return
-				}
-			}
-			lock.Unlock()
-		case err = <-errCh:
-			r.logger.Error("got an error from error channel", "error", err)
-			lock.Unlock()
-			return
-		default:
-			lock.Unlock()
-		}
-
-		r.logger.Debug("processed batch", "batches#", len(inputCh), "worker#", id)
-		statusMsg := fmt.Sprintf("converted to vector; batches: %d, worker#: %d", len(inputCh), id)
+		r.logger.Debug("wrote batch to db", "batch", batchCount, "size", len(nonEmptyBatch))
+		// Send progress status
+		statusMsg := fmt.Sprintf("processed batch %d/%d", batchCount, (len(paragraphs)+r.cfg.RAGBatchSize-1)/r.cfg.RAGBatchSize)
 		select {
 		case LongJobStatusCh <- statusMsg:
 		default:
-			r.logger.Warn("LongJobStatusCh channel full or closed, dropping status message", "message", statusMsg)
-			// Channel is full or closed, ignore the message to prevent panic
+			r.logger.Warn("LongJobStatusCh channel full, dropping message")
 		}
 	}
-}
-
-func (r *RAG) fetchEmb(lines []string, errCh chan error, vectorCh chan<- []models.VectorRow, slug, filename string) error {
-	// Filter out empty lines before sending to embedder
-	nonEmptyLines := make([]string, 0, len(lines))
-	for _, line := range lines {
-		trimmed := strings.TrimSpace(line)
-		if trimmed != "" {
-			nonEmptyLines = append(nonEmptyLines, trimmed)
-		}
+	r.logger.Debug("finished writing vectors", "batches", batchCount)
+	select {
+	case LongJobStatusCh <- FinishedRAGStatus:
+	default:
+		r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", FinishedRAGStatus)
 	}
-
-	// Skip if no non-empty lines
-	if len(nonEmptyLines) == 0 {
-		// Send empty result but don't error
-		vectorCh <- []models.VectorRow{}
-		return nil
-	}
-
-	embeddings, err := r.embedder.EmbedSlice(nonEmptyLines)
-	if err != nil {
-		r.logger.Error("failed to embed lines", "err", err.Error())
-		errCh <- err
-		return err
-	}
-
-	if len(embeddings) == 0 {
-		err := errors.New("no embeddings returned")
-		r.logger.Error("empty embeddings")
-		errCh <- err
-		return err
-	}
-
-	if len(embeddings) != len(nonEmptyLines) {
-		err := errors.New("mismatch between number of lines and embeddings returned")
-		r.logger.Error("embedding mismatch", "err", err.Error())
-		errCh <- err
-		return err
-	}
-
-	// Create a VectorRow for each line in the batch
-	vectors := make([]models.VectorRow, len(nonEmptyLines))
-	for i, line := range nonEmptyLines {
-		vectors[i] = models.VectorRow{
-			Embeddings: embeddings[i],
-			RawText:    line,
-			Slug:       fmt.Sprintf("%s_%d", slug, i),
-			FileName:   filename,
-		}
-	}
-
-	vectorCh <- vectors
 	return nil
 }
 

tables.go

@@ -236,9 +236,20 @@ func makeChatTable(chatMap map[string]models.Chat) *tview.Table {
 }
 
 // nolint:unused
+func formatSize(size int64) string {
+	units := []string{"B", "KB", "MB", "GB", "TB"}
+	i := 0
+	s := float64(size)
+	for s >= 1024 && i < len(units)-1 {
+		s /= 1024
+		i++
+	}
+	return fmt.Sprintf("%.1f%s", s, units[i])
+}
+
 func makeRAGTable(fileList []string) *tview.Flex {
 	actions := []string{"load", "delete"}
-	rows, cols := len(fileList), len(actions)+1
+	rows, cols := len(fileList), len(actions)+2
 	fileTable := tview.NewTable().
 		SetBorders(true)
 	longStatusView := tview.NewTextView()
@@ -252,39 +263,62 @@ func makeRAGTable(fileList []string) *tview.Flex {
 		AddItem(fileTable, 0, 60, true)
 	// Add the exit option as the first row (row 0)
 	fileTable.SetCell(0, 0,
-		tview.NewTableCell("Exit RAG manager").
+		tview.NewTableCell("File Name").
 			SetTextColor(tcell.ColorWhite).
 			SetAlign(tview.AlignCenter).
 			SetSelectable(false))
 	fileTable.SetCell(0, 1,
-		tview.NewTableCell("(Close without action)").
-			SetTextColor(tcell.ColorGray).
+		tview.NewTableCell("Preview").
+			SetTextColor(tcell.ColorWhite).
 			SetAlign(tview.AlignCenter).
 			SetSelectable(false))
 	fileTable.SetCell(0, 2,
-		tview.NewTableCell("exit").
-			SetTextColor(tcell.ColorGray).
-			SetAlign(tview.AlignCenter))
+		tview.NewTableCell("Load").
+			SetTextColor(tcell.ColorWhite).
+			SetAlign(tview.AlignCenter).
+			SetSelectable(false))
+	fileTable.SetCell(0, 3,
+		tview.NewTableCell("Delete").
+			SetTextColor(tcell.ColorWhite).
+			SetAlign(tview.AlignCenter).
+			SetSelectable(false))
 	// Add the file rows starting from row 1
 	for r := 0; r < rows; r++ {
 		for c := 0; c < cols; c++ {
 			color := tcell.ColorWhite
 			switch {
-			case c < 1:
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
+			case c == 0:
+				fileTable.SetCell(r+1, c,
 					tview.NewTableCell(fileList[r]).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter).
 						SetSelectable(false))
-			case c == 1: // Action description column - not selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell("(Action)").
+			case c == 1:
+				fpath := path.Join(cfg.RAGDir, fileList[r])
+				if fi, err := os.Stat(fpath); err == nil {
+					size := fi.Size()
+					modTime := fi.ModTime()
+					preview := fmt.Sprintf("%s | %s", formatSize(size), modTime.Format("2006-01-02 15:04"))
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell(preview).
+							SetTextColor(color).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				} else {
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell("error").
+							SetTextColor(color).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				}
+			case c == 2:
+				fileTable.SetCell(r+1, c,
+					tview.NewTableCell("load").
 						SetTextColor(color).
-						SetAlign(tview.AlignCenter).
-						SetSelectable(false))
-			default: // Action button column - selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell(actions[c-1]).
+						SetAlign(tview.AlignCenter))
+			default:
+				fileTable.SetCell(r+1, c,
+					tview.NewTableCell("delete").
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter))
 			}
@@ -318,7 +352,7 @@ func makeRAGTable(fileList []string) *tview.Flex {
 	}()
 	fileTable.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') || key == tcell.KeyCtrlX {
@@ -335,6 +369,8 @@ func makeRAGTable(fileList []string) *tview.Flex {
 		}
 		// defer pages.RemovePage(RAGPage)
 		tc := fileTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		fileTable.SetSelectable(false, false)
 		// Check if the selected row is the exit row (row 0) - do this first to avoid index issues
 		if row == 0 {
 			pages.RemovePage(RAGPage)
@@ -351,10 +387,15 @@ func makeRAGTable(fileList []string) *tview.Flex {
 				if err := ragger.LoadRAG(fpath); err != nil {
 					logger.Error("failed to embed file", "chat", fpath, "error", err)
 					_ = notifyUser("RAG", "failed to embed file; error: "+err.Error())
-					errCh <- err
-					// pages.RemovePage(RAGPage)
+					app.QueueUpdate(func() {
+						pages.RemovePage(RAGPage)
+					})
 					return
 				}
+				_ = notifyUser("RAG", "file loaded successfully")
+				app.QueueUpdate(func() {
+					pages.RemovePage(RAGPage)
+				})
 			}()
 			return
 		case "delete":
@@ -385,7 +426,7 @@ func makeRAGTable(fileList []string) *tview.Flex {
 
 func makeLoadedRAGTable(fileList []string) *tview.Flex {
 	actions := []string{"delete"}
-	rows, cols := len(fileList), len(actions)+1
+	rows, cols := len(fileList), len(actions)+2
 	// Add 1 extra row for the "exit" option at the top
 	fileTable := tview.NewTable().
 		SetBorders(true)
@@ -400,39 +441,61 @@ func makeLoadedRAGTable(fileList []string) *tview.Flex {
 		AddItem(fileTable, 0, 60, true)
 	// Add the exit option as the first row (row 0)
 	fileTable.SetCell(0, 0,
-		tview.NewTableCell("Exit Loaded Files manager").
+		tview.NewTableCell("File Name").
 			SetTextColor(tcell.ColorWhite).
 			SetAlign(tview.AlignCenter).
 			SetSelectable(false))
 	fileTable.SetCell(0, 1,
-		tview.NewTableCell("(Close without action)").
-			SetTextColor(tcell.ColorGray).
+		tview.NewTableCell("Preview").
+			SetTextColor(tcell.ColorWhite).
 			SetAlign(tview.AlignCenter).
 			SetSelectable(false))
 	fileTable.SetCell(0, 2,
-		tview.NewTableCell("exit").
-			SetTextColor(tcell.ColorGray).
-			SetAlign(tview.AlignCenter))
+		tview.NewTableCell("Load").
+			SetTextColor(tcell.ColorWhite).
+			SetAlign(tview.AlignCenter).
+			SetSelectable(false))
+	fileTable.SetCell(0, 3,
+		tview.NewTableCell("Delete").
+			SetTextColor(tcell.ColorWhite).
+			SetAlign(tview.AlignCenter).
+			SetSelectable(false))
 	// Add the file rows starting from row 1
 	for r := 0; r < rows; r++ {
 		for c := 0; c < cols; c++ {
 			color := tcell.ColorWhite
 			switch {
-			case c < 1:
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
+			case c == 0:
+				fileTable.SetCell(r+1, c,
 					tview.NewTableCell(fileList[r]).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter).
 						SetSelectable(false))
-			case c == 1: // Action description column - not selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell("(Action)").
+			case c == 1:
+				if fi, err := os.Stat(fileList[r]); err == nil {
+					size := fi.Size()
+					modTime := fi.ModTime()
+					preview := fmt.Sprintf("%s | %s", formatSize(size), modTime.Format("2006-01-02 15:04"))
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell(preview).
+							SetTextColor(color).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				} else {
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell("error").
+							SetTextColor(color).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				}
+			case c == 2:
+				fileTable.SetCell(r+1, c,
+					tview.NewTableCell("load").
 						SetTextColor(color).
-						SetAlign(tview.AlignCenter).
-						SetSelectable(false))
-			default: // Action button column - selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell(actions[c-1]).
+						SetAlign(tview.AlignCenter))
+			default:
+				fileTable.SetCell(r+1, c,
+					tview.NewTableCell("delete").
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter))
 			}
@@ -440,7 +503,7 @@ func makeLoadedRAGTable(fileList []string) *tview.Flex {
 	}
 	fileTable.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') || key == tcell.KeyCtrlX {
@@ -456,6 +519,8 @@ func makeLoadedRAGTable(fileList []string) *tview.Flex {
 			return
 		}
 		tc := fileTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		fileTable.SetSelectable(false, false)
 		// Check if the selected row is the exit row (row 0) - do this first to avoid index issues
 		if row == 0 {
 			pages.RemovePage(RAGLoadedPage)
@@ -533,7 +598,7 @@ func makeAgentTable(agentList []string) *tview.Table {
 	}
 	chatActTable.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') {
@@ -549,6 +614,8 @@ func makeAgentTable(agentList []string) *tview.Table {
 			return
 		}
 		tc := chatActTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		chatActTable.SetSelectable(false, false)
 		selected := agentList[row]
 		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
 		switch tc.Text {
@@ -630,7 +697,7 @@ func makeCodeBlockTable(codeBlocks []string) *tview.Table {
 	}
 	table.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') {
@@ -646,6 +713,8 @@ func makeCodeBlockTable(codeBlocks []string) *tview.Table {
 			return
 		}
 		tc := table.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		table.SetSelectable(false, false)
 		selected := codeBlocks[row]
 		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
 		switch tc.Text {
@@ -702,7 +771,7 @@ func makeImportChatTable(filenames []string) *tview.Table {
 	}
 	chatActTable.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') {
@@ -718,6 +787,8 @@ func makeImportChatTable(filenames []string) *tview.Table {
 			return
 		}
 		tc := chatActTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		chatActTable.SetSelectable(false, false)
 		selected := filenames[row]
 		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
 		switch tc.Text {

tui.go

@@ -15,6 +15,11 @@ import (
 	"github.com/rivo/tview"
 )
 
+func isFullScreenPageActive() bool {
+	name, _ := pages.GetFrontPage()
+	return name != "main"
+}
+
 var (
 	app                *tview.Application
 	pages              *tview.Pages
@@ -525,6 +530,9 @@ func init() {
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'i' && event.Modifiers()&tcell.ModAlt != 0 {
+			if isFullScreenPageActive() {
+				return event
+			}
 			showColorschemeSelectionPopup()
 			return nil
 		}
@@ -731,6 +739,9 @@ func init() {
 			return nil
 		}
 		if event.Key() == tcell.KeyCtrlL {
+			if isFullScreenPageActive() {
+				return event
+			}
 			// Show model selection popup instead of rotating models
 			showModelSelectionPopup()
 			return nil
@@ -744,6 +755,9 @@ func init() {
 			return nil
 		}
 		if event.Key() == tcell.KeyCtrlV {
+			if isFullScreenPageActive() {
+				return event
+			}
 			// Show API link selection popup instead of rotating APIs
 			showAPILinkSelectionPopup()
 			return nil
@@ -850,11 +864,17 @@ func init() {
 			return nil
 		}
 		if event.Key() == tcell.KeyCtrlQ {
+			if isFullScreenPageActive() {
+				return event
+			}
 			// Show user role selection popup instead of cycling through roles
 			showUserRoleSelectionPopup()
 			return nil
 		}
 		if event.Key() == tcell.KeyCtrlX {
+			if isFullScreenPageActive() {
+				return event
+			}
 			// Show bot role selection popup instead of cycling through roles
 			showBotRoleSelectionPopup()
 			return nil
@@ -975,13 +995,6 @@ func init() {
 				}
 				// go chatRound(msgText, persona, textView, false, false)
 				chatRoundChan <- &models.ChatRoundReq{Role: persona, UserMsg: msgText}
-				// Also clear any image attachment after sending the message
-				go func() {
-					// Wait a short moment for the message to be processed, then clear the image attachment
-					// This allows the image to be sent with the current message if it was attached
-					// But clears it for the next message
-					ClearImageAttachment()
-				}()
 			}
 			return nil
 		}