Enha(tools.todo): always provide whole todo list

logic requires reference to config

Makefile

@@ -1,4 +1,4 @@
-.PHONY: setconfig run lint setup-whisper build-whisper download-whisper-model docker-up docker-down docker-logs noextra-run
+.PHONY: setconfig run lint lintall install-linters setup-whisper build-whisper download-whisper-model docker-up docker-down docker-logs noextra-run installdelve checkdelve
 
 run: setconfig
 	go build -tags extra -o gf-lt && ./gf-lt
@@ -15,9 +15,21 @@ noextra-run: setconfig
 setconfig:
 	find config.toml &>/dev/null || cp config.example.toml config.toml
 
+installdelve:
+	go install github.com/go-delve/delve/cmd/dlv@latest
+
+checkdelve:
+	which dlv &>/dev/null || installdelve
+
+install-linters: ## Install additional linters (noblanks)
+	go install github.com/GrailFinder/noblanks-linter/cmd/noblanks@latest
+
 lint: ## Run linters. Use make install-linters first.
 	golangci-lint run -c .golangci.yml ./...
 
+lintall: lint
+	noblanks ./...
+
 # Whisper STT Setup (in batteries directory)
 setup-whisper: build-whisper download-whisper-model
 

agent/request.go

@@ -71,8 +71,8 @@ func (ag *AgentClient) buildRequest(sysprompt, msg string) ([]byte, error) {
 	// Build prompt for completion endpoints
 	if isCompletion {
 		var sb strings.Builder
-		for _, m := range messages {
+		for i := range messages {
-			sb.WriteString(m.ToPrompt())
+			sb.WriteString(messages[i].ToPrompt())
 			sb.WriteString("\n")
 		}
 		prompt := strings.TrimSpace(sb.String())
@@ -140,7 +140,6 @@ func (ag *AgentClient) LLMRequest(body io.Reader) ([]byte, error) {
 		ag.log.Error("failed to read request body", "error", err)
 		return nil, err
 	}
-
 	req, err := http.NewRequest("POST", ag.cfg.CurrentAPI, bytes.NewReader(bodyBytes))
 	if err != nil {
 		ag.log.Error("failed to create request", "error", err)
@@ -150,22 +149,18 @@ func (ag *AgentClient) LLMRequest(body io.Reader) ([]byte, error) {
 	req.Header.Add("Content-Type", "application/json")
 	req.Header.Add("Authorization", "Bearer "+ag.getToken())
 	req.Header.Set("Accept-Encoding", "gzip")
-
 	ag.log.Debug("agent LLM request", "url", ag.cfg.CurrentAPI, "body_preview", string(bodyBytes[:min(len(bodyBytes), 500)]))
-
 	resp, err := httpClient.Do(req)
 	if err != nil {
 		ag.log.Error("llamacpp api request failed", "error", err, "url", ag.cfg.CurrentAPI)
 		return nil, err
 	}
 	defer resp.Body.Close()
-
 	responseBytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		ag.log.Error("failed to read response", "error", err)
 		return nil, err
 	}
-
 	if resp.StatusCode >= 400 {
 		ag.log.Error("agent LLM request failed", "status", resp.StatusCode, "response", string(responseBytes[:min(len(responseBytes), 1000)]))
 		return responseBytes, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(responseBytes[:min(len(responseBytes), 200)]))
@@ -178,7 +173,6 @@ func (ag *AgentClient) LLMRequest(body io.Reader) ([]byte, error) {
 		// Return raw response as fallback
 		return responseBytes, nil
 	}
-
 	return []byte(text), nil
 }
 

bot.go

@@ -23,8 +23,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-
-	"github.com/neurosnap/sentences/english"
 )
 
 var (
@@ -46,6 +44,7 @@ var (
 	ragger          *rag.RAG
 	chunkParser     ChunkParser
 	lastToolCall    *models.FuncCall
+	lastRespStats   *models.ResponseStats
 	//nolint:unused // TTS_ENABLED conditionally uses this
 	orator          Orator
 	asr             STT
@@ -67,6 +66,8 @@ var (
 	LocalModels = []string{}
 )
 
+var thinkBlockRE = regexp.MustCompile(`(?s)<think>.*?</think>`)
+
 // parseKnownToTag extracts known_to list from content using configured tag.
 // Returns cleaned content and list of character names.
 func parseKnownToTag(content string) []string {
@@ -118,7 +119,7 @@ func processMessageTag(msg *models.RoleMsg) *models.RoleMsg {
 	}
 	// If KnownTo already set, assume tag already processed (content cleaned).
 	// However, we still check for new tags (maybe added later).
-	knownTo := parseKnownToTag(msg.Content)
+	knownTo := parseKnownToTag(msg.GetText())
 	// If tag found, replace KnownTo with new list (merge with existing?)
 	// For simplicity, if knownTo is not nil, replace.
 	if knownTo == nil {
@@ -137,6 +138,9 @@ func processMessageTag(msg *models.RoleMsg) *models.RoleMsg {
 // filterMessagesForCharacter returns messages visible to the specified character.
 // If CharSpecificContextEnabled is false, returns all messages.
 func filterMessagesForCharacter(messages []models.RoleMsg, character string) []models.RoleMsg {
+	if strings.Contains(cfg.CurrentAPI, "chat") {
+		return messages
+	}
 	if cfg == nil || !cfg.CharSpecificContextEnabled || character == "" {
 		return messages
 	}
@@ -144,97 +148,67 @@ func filterMessagesForCharacter(messages []models.RoleMsg, character string) []m
 		return messages
 	}
 	filtered := make([]models.RoleMsg, 0, len(messages))
-	for _, msg := range messages {
+	for i := range messages {
 		// If KnownTo is nil or empty, message is visible to all
 		// system msg cannot be filtered
-		if len(msg.KnownTo) == 0 || msg.Role == "system" {
+		if len(messages[i].KnownTo) == 0 || messages[i].Role == "system" {
-			filtered = append(filtered, msg)
+			filtered = append(filtered, messages[i])
 			continue
 		}
-		if slices.Contains(msg.KnownTo, character) {
+		if slices.Contains(messages[i].KnownTo, character) {
 			// Check if character is in KnownTo lis
-			filtered = append(filtered, msg)
+			filtered = append(filtered, messages[i])
 		}
 	}
 	return filtered
 }
 
-func cleanToolCalls(messages []models.RoleMsg) []models.RoleMsg {
-	// If AutoCleanToolCallsFromCtx is false, keep tool call messages in context
-	if cfg != nil && !cfg.AutoCleanToolCallsFromCtx {
-		return consolidateAssistantMessages(messages)
-	}
-	cleaned := make([]models.RoleMsg, 0, len(messages))
-	for i, msg := range messages {
-		// recognize the message as the tool call and remove it
-		// tool call in last msg should stay
-		if msg.ToolCallID == "" || i == len(messages)-1 {
-			cleaned = append(cleaned, msg)
-		}
-	}
-	return consolidateAssistantMessages(cleaned)
-}
-
-// consolidateAssistantMessages merges consecutive assistant messages into a single message
 func consolidateAssistantMessages(messages []models.RoleMsg) []models.RoleMsg {
 	if len(messages) == 0 {
 		return messages
 	}
-	consolidated := make([]models.RoleMsg, 0, len(messages))
+	result := make([]models.RoleMsg, 0, len(messages))
-	currentAssistantMsg := models.RoleMsg{}
+	for i := range messages {
-	isBuildingAssistantMsg := false
+		// Non-assistant messages are appended as-is
-	for i := 0; i < len(messages); i++ {
+		if messages[i].Role != cfg.AssistantRole {
-		msg := messages[i]
+			result = append(result, messages[i])
-		// assistant role only
+			continue
-		if msg.Role == cfg.AssistantRole {
+		}
-			// If this is an assistant message, start or continue building
+		// Assistant message: start a new block or merge with the last one
-			if !isBuildingAssistantMsg {
+		if len(result) == 0 || result[len(result)-1].Role != cfg.AssistantRole {
-				// Start accumulating assistant message
+			// First assistant in a block: append a copy (avoid mutating input)
-				currentAssistantMsg = msg.Copy()
+			result = append(result, messages[i].Copy())
-				isBuildingAssistantMsg = true
+			continue
-			} else {
+		}
-				// Continue accumulating - append content to the current assistant message
+		// Merge with the last assistant message
-				if currentAssistantMsg.IsContentParts() || msg.IsContentParts() {
+		last := &result[len(result)-1]
-					// Handle structured content
+		// If either message has structured content, unify to ContentParts
-					if !currentAssistantMsg.IsContentParts() {
+		if last.IsContentParts() || messages[i].IsContentParts() {
-						// Preserve the original ToolCallID before conversion
+			// Convert last to ContentParts if needed, preserving ToolCallID
-						originalToolCallID := currentAssistantMsg.ToolCallID
+			if !last.IsContentParts() {
-						// Convert existing content to content parts
+				toolCallID := last.ToolCallID
-						currentAssistantMsg = models.NewMultimodalMsg(currentAssistantMsg.Role, []interface{}{models.TextContentPart{Type: "text", Text: currentAssistantMsg.Content}})
+				*last = models.NewMultimodalMsg(last.Role, []interface{}{
-						// Restore the original ToolCallID to preserve tool call linking
+					models.TextContentPart{Type: "text", Text: last.Content},
-						currentAssistantMsg.ToolCallID = originalToolCallID
+				})
-					}
+				last.ToolCallID = toolCallID
-					if msg.IsContentParts() {
+			}
-						currentAssistantMsg.ContentParts = append(currentAssistantMsg.ContentParts, msg.GetContentParts()...)
+			// Add current message's content to last
-					} else if msg.Content != "" {
+			if messages[i].IsContentParts() {
-						currentAssistantMsg.AddTextPart(msg.Content)
+				last.ContentParts = append(last.ContentParts, messages[i].GetContentParts()...)
-					}
+			} else if messages[i].Content != "" {
-				} else {
+				last.AddTextPart(messages[i].Content)
-					// Simple string content
-					if currentAssistantMsg.Content != "" {
-						currentAssistantMsg.Content += "\n" + msg.Content
-					} else {
-						currentAssistantMsg.Content = msg.Content
-					}
-					// ToolCallID is already preserved since we're not creating a new message object when just concatenating content
-				}
 			}
 		} else {
-			// This is not an assistant message
+			// Both simple strings: concatenate with newline
-			// If we were building an assistant message, add it to the result
+			if last.Content != "" && messages[i].Content != "" {
-			if isBuildingAssistantMsg {
+				last.Content += "\n" + messages[i].Content
-				consolidated = append(consolidated, currentAssistantMsg)
+			} else if messages[i].Content != "" {
-				isBuildingAssistantMsg = false
+				last.Content = messages[i].Content
 			}
-			// Add the non-assistant message
+			// ToolCallID is already preserved in last
-			consolidated = append(consolidated, msg)
 		}
 	}
-	// Don't forget the last assistant message if we were building one
+	return result
-	if isBuildingAssistantMsg {
-		consolidated = append(consolidated, currentAssistantMsg)
-	}
-	return consolidated
 }
 
 // GetLogLevel returns the current log level as a string
@@ -410,14 +384,21 @@ func fetchLCPModelsWithLoadStatus() ([]string, error) {
 		return nil, err
 	}
 	result := make([]string, 0, len(models.Data))
-	for _, m := range models.Data {
+	li := 0 // loaded index
+	for i, m := range models.Data {
 		modelName := m.ID
 		if m.Status.Value == "loaded" {
 			modelName = "(loaded) " + modelName
+			li = i
 		}
 		result = append(result, modelName)
 	}
-	return result, nil
+	if li == 0 {
+		return result, nil // no loaded models
+	}
+	loadedModel := result[li]
+	result = append(result[:li], result[li+1:]...)
+	return slices.Concat([]string{loadedModel}, result), nil
 }
 
 // fetchLCPModelsWithStatus returns the full LCPModels struct including status information.
@@ -484,30 +465,28 @@ func monitorModelLoad(modelID string) {
 // extractDetailedErrorFromBytes extracts detailed error information from response body bytes
 func extractDetailedErrorFromBytes(body []byte, statusCode int) string {
 	// Try to parse as JSON to extract detailed error information
-	var errorResponse map[string]interface{}
+	var errorResponse map[string]any
 	if err := json.Unmarshal(body, &errorResponse); err == nil {
 		// Check if it's an error response with detailed information
 		if errorData, ok := errorResponse["error"]; ok {
-			if errorMap, ok := errorData.(map[string]interface{}); ok {
+			if errorMap, ok := errorData.(map[string]any); ok {
 				var errorMsg string
 				if msg, ok := errorMap["message"]; ok {
 					errorMsg = fmt.Sprintf("%v", msg)
 				}
-
 				var details []string
 				if code, ok := errorMap["code"]; ok {
 					details = append(details, fmt.Sprintf("Code: %v", code))
 				}
-
 				if metadata, ok := errorMap["metadata"]; ok {
 					// Handle metadata which might contain raw error details
-					if metadataMap, ok := metadata.(map[string]interface{}); ok {
+					if metadataMap, ok := metadata.(map[string]any); ok {
 						if raw, ok := metadataMap["raw"]; ok {
 							// Parse the raw error string if it's JSON
-							var rawError map[string]interface{}
+							var rawError map[string]any
 							if rawStr, ok := raw.(string); ok && json.Unmarshal([]byte(rawStr), &rawError) == nil {
 								if rawErrorData, ok := rawError["error"]; ok {
-									if rawErrorMap, ok := rawErrorData.(map[string]interface{}); ok {
+									if rawErrorMap, ok := rawErrorData.(map[string]any); ok {
 										if rawMsg, ok := rawErrorMap["message"]; ok {
 											return fmt.Sprintf("API Error: %s", rawMsg)
 										}
@@ -518,20 +497,30 @@ func extractDetailedErrorFromBytes(body []byte, statusCode int) string {
 					}
 					details = append(details, fmt.Sprintf("Metadata: %v", metadata))
 				}
-
 				if len(details) > 0 {
 					return fmt.Sprintf("API Error: %s (%s)", errorMsg, strings.Join(details, ", "))
 				}
-
 				return "API Error: " + errorMsg
 			}
 		}
 	}
-
 	// If not a structured error response, return the raw body with status
 	return fmt.Sprintf("HTTP Status: %d, Response Body: %s", statusCode, string(body))
 }
 
+func finalizeRespStats(tokenCount int, startTime time.Time) {
+	duration := time.Since(startTime).Seconds()
+	var tps float64
+	if duration > 0 {
+		tps = float64(tokenCount) / duration
+	}
+	lastRespStats = &models.ResponseStats{
+		Tokens:       tokenCount,
+		Duration:     duration,
+		TokensPerSec: tps,
+	}
+}
+
 // sendMsgToLLM expects streaming resp
 func sendMsgToLLM(body io.Reader) {
 	choseChunkParser()
@@ -560,7 +549,6 @@ func sendMsgToLLM(body io.Reader) {
 		streamDone <- true
 		return
 	}
-
 	// Check if the initial response is an error before starting to stream
 	if resp.StatusCode >= 400 {
 		// Read the response body to get detailed error information
@@ -575,7 +563,6 @@ func sendMsgToLLM(body io.Reader) {
 			streamDone <- true
 			return
 		}
-
 		// Parse the error response for detailed information
 		detailedError := extractDetailedErrorFromBytes(bodyBytes, resp.StatusCode)
 		logger.Error("API returned error status", "status_code", resp.StatusCode, "detailed_error", detailedError)
@@ -586,12 +573,17 @@ func sendMsgToLLM(body io.Reader) {
 		streamDone <- true
 		return
 	}
-
+	//
 	defer resp.Body.Close()
 	reader := bufio.NewReader(resp.Body)
 	counter := uint32(0)
+	tokenCount := 0
+	startTime := time.Now()
 	hasReasoning := false
 	reasoningSent := false
+	defer func() {
+		finalizeRespStats(tokenCount, startTime)
+	}()
 	for {
 		var (
 			answerText string
@@ -667,11 +659,13 @@ func sendMsgToLLM(body io.Reader) {
 			// Close the thinking block if we were streaming reasoning and haven't closed it yet
 			if hasReasoning && !reasoningSent {
 				chunkChan <- "</think>"
+				tokenCount++
 			}
 			if chunk.Chunk != "" {
 				logger.Warn("text inside of finish llmchunk", "chunk", chunk, "counter", counter)
 				answerText = strings.ReplaceAll(chunk.Chunk, "\n\n", "\n")
 				chunkChan <- answerText
+				tokenCount++
 			}
 			streamDone <- true
 			break
@@ -684,22 +678,23 @@ func sendMsgToLLM(body io.Reader) {
 			if !hasReasoning {
 				// First reasoning chunk - send opening tag
 				chunkChan <- "<think>"
+				tokenCount++
 				hasReasoning = true
 			}
 			// Stream reasoning content immediately
 			answerText = strings.ReplaceAll(chunk.Reasoning, "\n\n", "\n")
 			if answerText != "" {
 				chunkChan <- answerText
+				tokenCount++
 			}
 		}
-
 		// When we get content and have been streaming reasoning, close the thinking block
 		if chunk.Chunk != "" && hasReasoning && !reasoningSent {
 			// Close the thinking block before sending actual content
 			chunkChan <- "</think>"
+			tokenCount++
 			reasoningSent = true
 		}
-
 		// bot sends way too many \n
 		answerText = strings.ReplaceAll(chunk.Chunk, "\n\n", "\n")
 		// Accumulate text to check for stop strings that might span across chunks
@@ -709,9 +704,11 @@ func sendMsgToLLM(body io.Reader) {
 			slices.Contains(stopStrings, answerText) {
 			logger.Debug("stop string detected on client side for completion endpoint", "stop_string", answerText)
 			streamDone <- true
+			break
 		}
 		if answerText != "" {
 			chunkChan <- answerText
+			tokenCount++
 		}
 		openAIToolChan <- chunk.ToolChunk
 		if chunk.FuncName != "" {
@@ -729,68 +726,6 @@ func sendMsgToLLM(body io.Reader) {
 	}
 }
 
-func chatRagUse(qText string) (string, error) {
-	logger.Debug("Starting RAG query", "original_query", qText)
-	tokenizer, err := english.NewSentenceTokenizer(nil)
-	if err != nil {
-		logger.Error("failed to create sentence tokenizer", "error", err)
-		return "", err
-	}
-	// this where llm should find the questions in text and ask them
-	questionsS := tokenizer.Tokenize(qText)
-	questions := make([]string, len(questionsS))
-	for i, q := range questionsS {
-		questions[i] = q.Text
-		logger.Debug("RAG question extracted", "index", i, "question", q.Text)
-	}
-
-	if len(questions) == 0 {
-		logger.Warn("No questions extracted from query text", "query", qText)
-		return "No related results from RAG vector storage.", nil
-	}
-
-	respVecs := []models.VectorRow{}
-	for i, q := range questions {
-		logger.Debug("Processing RAG question", "index", i, "question", q)
-		emb, err := ragger.LineToVector(q)
-		if err != nil {
-			logger.Error("failed to get embeddings for RAG", "error", err, "index", i, "question", q)
-			continue
-		}
-		logger.Debug("Got embeddings for question", "index", i, "question_len", len(q), "embedding_len", len(emb))
-
-		// Create EmbeddingResp struct for the search
-		embeddingResp := &models.EmbeddingResp{
-			Embedding: emb,
-			Index:     0, // Not used in search but required for the struct
-		}
-		vecs, err := ragger.SearchEmb(embeddingResp)
-		if err != nil {
-			logger.Error("failed to query embeddings in RAG", "error", err, "index", i, "question", q)
-			continue
-		}
-		logger.Debug("RAG search returned vectors", "index", i, "question", q, "vector_count", len(vecs))
-		respVecs = append(respVecs, vecs...)
-	}
-
-	// get raw text
-	resps := []string{}
-	logger.Debug("RAG query final results", "total_vecs_found", len(respVecs))
-	for _, rv := range respVecs {
-		resps = append(resps, rv.RawText)
-		logger.Debug("RAG result", "slug", rv.Slug, "filename", rv.FileName, "raw_text_len", len(rv.RawText))
-	}
-
-	if len(resps) == 0 {
-		logger.Info("No RAG results found for query", "original_query", qText, "question_count", len(questions))
-		return "No related results from RAG vector storage.", nil
-	}
-
-	result := strings.Join(resps, "\n")
-	logger.Debug("RAG query completed", "result_len", len(result), "response_count", len(resps))
-	return result, nil
-}
-
 func roleToIcon(role string) string {
 	return "<" + role + ">: "
 }
@@ -808,14 +743,46 @@ func chatWatcher(ctx context.Context) {
 	}
 }
 
+// inpired by https://github.com/rivo/tview/issues/225
+func showSpinner() {
+	spinners := []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"}
+	var i int
+	botPersona := cfg.AssistantRole
+	if cfg.WriteNextMsgAsCompletionAgent != "" {
+		botPersona = cfg.WriteNextMsgAsCompletionAgent
+	}
+	for botRespMode || toolRunningMode {
+		time.Sleep(400 * time.Millisecond)
+		spin := i % len(spinners)
+		app.QueueUpdateDraw(func() {
+			switch {
+			case toolRunningMode:
+				textArea.SetTitle(spinners[spin] + " tool")
+			case botRespMode:
+				textArea.SetTitle(spinners[spin] + " " + botPersona + " (F6 to interrupt)")
+			default:
+				textArea.SetTitle(spinners[spin] + " input")
+			}
+		})
+		i++
+	}
+	app.QueueUpdateDraw(func() {
+		textArea.SetTitle("input")
+	})
+}
+
 func chatRound(r *models.ChatRoundReq) error {
 	botRespMode = true
+	go showSpinner()
 	updateStatusLine()
 	botPersona := cfg.AssistantRole
 	if cfg.WriteNextMsgAsCompletionAgent != "" {
 		botPersona = cfg.WriteNextMsgAsCompletionAgent
 	}
-	defer func() { botRespMode = false }()
+	defer func() {
+		botRespMode = false
+		ClearImageAttachment()
+	}()
 	// check that there is a model set to use if is not local
 	choseChunkParser()
 	reader, err := chunkParser.FormMsg(r.UserMsg, r.Role, r.Resume)
@@ -834,13 +801,14 @@ func chatRound(r *models.ChatRoundReq) error {
 		chatBody.Messages = append(chatBody.Messages, models.RoleMsg{
 			Role: botPersona, Content: "",
 		})
-		fmt.Fprintf(textView, "\n[-:-:b](%d) ", msgIdx)
+		nl := "\n\n"
-		fmt.Fprint(textView, roleToIcon(botPersona))
+		prevText := textView.GetText(true)
-		fmt.Fprint(textView, "[-:-:-]\n")
+		if strings.HasSuffix(prevText, nl) {
-		if cfg.ThinkUse && !strings.Contains(cfg.CurrentAPI, "v1") {
+			nl = ""
-			// fmt.Fprint(textView, "<think>")
+		} else if strings.HasSuffix(prevText, "\n") {
-			chunkChan <- "<think>"
+			nl = "\n"
 		}
+		fmt.Fprintf(textView, "%s[-:-:b](%d) %s[-:-:-]\n", nl, msgIdx, roleToIcon(botPersona))
 	} else {
 		msgIdx = len(chatBody.Messages) - 1
 	}
@@ -914,7 +882,6 @@ out:
 				textView.ScrollToEnd()
 			}
 		case <-streamDone:
-			// drain any remaining chunks from chunkChan before exiting
 			for len(chunkChan) > 0 {
 				chunk := <-chunkChan
 				fmt.Fprint(textView, chunk)
@@ -923,31 +890,40 @@ out:
 					textView.ScrollToEnd()
 				}
 				if cfg.TTS_ENABLED {
-					// Send chunk to audio stream handler
 					TTSTextChan <- chunk
 				}
 			}
 			if cfg.TTS_ENABLED {
-				// msg is done; flush it down
 				TTSFlushChan <- true
 			}
 			break out
 		}
 	}
+	var msgStats *models.ResponseStats
+	if lastRespStats != nil {
+		msgStats = &models.ResponseStats{
+			Tokens:       lastRespStats.Tokens,
+			Duration:     lastRespStats.Duration,
+			TokensPerSec: lastRespStats.TokensPerSec,
+		}
+		lastRespStats = nil
+	}
 	botRespMode = false
-	// numbers in chatbody and displayed must be the same
 	if r.Resume {
 		chatBody.Messages[len(chatBody.Messages)-1].Content += respText.String()
-		// lastM.Content = lastM.Content + respText.String()
-		// Process the updated message to check for known_to tags in resumed response
 		updatedMsg := chatBody.Messages[len(chatBody.Messages)-1]
 		processedMsg := processMessageTag(&updatedMsg)
 		chatBody.Messages[len(chatBody.Messages)-1] = *processedMsg
+		if msgStats != nil && chatBody.Messages[len(chatBody.Messages)-1].Role != cfg.ToolRole {
+			chatBody.Messages[len(chatBody.Messages)-1].Stats = msgStats
+		}
 	} else {
-		// Message was already added at the start, just process it for known_to tags
 		chatBody.Messages[msgIdx].Content = respText.String()
 		processedMsg := processMessageTag(&chatBody.Messages[msgIdx])
 		chatBody.Messages[msgIdx] = *processedMsg
+		if msgStats != nil && chatBody.Messages[msgIdx].Role != cfg.ToolRole {
+			chatBody.Messages[msgIdx].Stats = msgStats
+		}
 		stopTTSIfNotForUser(&chatBody.Messages[msgIdx])
 	}
 	cleanChatBody()
@@ -959,7 +935,9 @@ out:
 	if err := updateStorageChat(activeChatName, chatBody.Messages); err != nil {
 		logger.Warn("failed to update storage", "error", err, "name", activeChatName)
 	}
-	if findCall(respText.String(), toolResp.String()) {
+	// Strip think blocks before parsing for tool calls
+	respTextNoThink := thinkBlockRE.ReplaceAllString(respText.String(), "")
+	if findCall(respTextNoThink, toolResp.String()) {
 		return nil
 	}
 	// Check if this message was sent privately to specific characters
@@ -981,7 +959,7 @@ func cleanChatBody() {
 	}
 	// Tool request cleaning is now configurable via AutoCleanToolCallsFromCtx (default false)
 	// /completion msg where part meant for user and other part tool call
-	chatBody.Messages = cleanToolCalls(chatBody.Messages)
+	// chatBody.Messages = cleanToolCalls(chatBody.Messages)
 	chatBody.Messages = consolidateAssistantMessages(chatBody.Messages)
 }
 
@@ -1095,22 +1073,38 @@ func findCall(msg, toolCall string) bool {
 		}
 		lastToolCall.Args = openAIToolMap
 		fc = lastToolCall
-		// Set lastToolCall.ID from parsed tool call ID if available
+		// NOTE: We do NOT override lastToolCall.ID from arguments.
-		if len(openAIToolMap) > 0 {
+		// The ID should come from the streaming response (chunk.ToolID) set earlier.
-			if id, exists := openAIToolMap["id"]; exists {
+		// Some tools like todo_create have "id" in their arguments which is NOT the tool call ID.
-				lastToolCall.ID = id
-			}
-		}
 	} else {
 		jsStr := toolCallRE.FindString(msg)
 		if jsStr == "" { // no tool call case
 			return false
 		}
-		prefix := "__tool_call__\n"
+		// Remove prefix/suffix with flexible whitespace handling
-		suffix := "\n__tool_call__"
+		jsStr = strings.TrimSpace(jsStr)
-		jsStr = strings.TrimSuffix(strings.TrimPrefix(jsStr, prefix), suffix)
+		jsStr = strings.TrimPrefix(jsStr, "__tool_call__")
+		jsStr = strings.TrimSuffix(jsStr, "__tool_call__")
+		jsStr = strings.TrimSpace(jsStr)
 		// HTML-decode the JSON string to handle encoded characters like &lt; -> <=
 		decodedJsStr := html.UnescapeString(jsStr)
+		// Try to find valid JSON bounds (first { to last })
+		start := strings.Index(decodedJsStr, "{")
+		end := strings.LastIndex(decodedJsStr, "}")
+		if start == -1 || end == -1 || end <= start {
+			logger.Error("failed to find valid JSON in tool call", "json_string", decodedJsStr)
+			toolResponseMsg := models.RoleMsg{
+				Role:    cfg.ToolRole,
+				Content: "Error processing tool call: no valid JSON found. Please check the JSON format.",
+			}
+			chatBody.Messages = append(chatBody.Messages, toolResponseMsg)
+			crr := &models.ChatRoundReq{
+				Role: cfg.AssistantRole,
+			}
+			chatRoundChan <- crr
+			return true
+		}
+		decodedJsStr = decodedJsStr[start : end+1]
 		var err error
 		fc, err = unmarshalFuncCall(decodedJsStr)
 		if err != nil {
@@ -1137,14 +1131,18 @@ func findCall(msg, toolCall string) bool {
 		lastToolCall.Args = fc.Args
 	}
 	// we got here => last msg recognized as a tool call (correct or not)
-	// make sure it has ToolCallID
+	// Use the tool call ID from streaming response (lastToolCall.ID)
-	if chatBody.Messages[len(chatBody.Messages)-1].ToolCallID == "" {
+	// Don't generate random ID - the ID should match between assistant message and tool response
-		// Tool call IDs should be alphanumeric strings with length 9!
+	lastMsgIdx := len(chatBody.Messages) - 1
-		chatBody.Messages[len(chatBody.Messages)-1].ToolCallID = randString(9)
+	if lastToolCall.ID != "" {
+		chatBody.Messages[lastMsgIdx].ToolCallID = lastToolCall.ID
 	}
-	// Ensure lastToolCall.ID is set, fallback to assistant message's ToolCallID
+	// Store tool call info in the assistant message
-	if lastToolCall.ID == "" {
+	// Convert Args map to JSON string for storage
-		lastToolCall.ID = chatBody.Messages[len(chatBody.Messages)-1].ToolCallID
+	chatBody.Messages[lastMsgIdx].ToolCall = &models.ToolCall{
+		ID:   lastToolCall.ID,
+		Name: lastToolCall.Name,
+		Args: mapToString(lastToolCall.Args),
 	}
 	// call a func
 	_, ok := fnMap[fc.Name]
@@ -1169,16 +1167,23 @@ func findCall(msg, toolCall string) bool {
 		chatRoundChan <- crr
 		return true
 	}
+	// Show tool call progress indicator before execution
+	fmt.Fprintf(textView, "\n[yellow::i][tool: %s...][-:-:-]", fc.Name)
+	toolRunningMode = true
 	resp := callToolWithAgent(fc.Name, fc.Args)
-	toolMsg := string(resp) // Remove the "tool response: " prefix and %+v formatting
+	toolRunningMode = false
+	toolMsg := string(resp)
 	logger.Info("llm used a tool call", "tool_name", fc.Name, "too_args", fc.Args, "id", fc.ID, "tool_resp", toolMsg)
 	fmt.Fprintf(textView, "%s[-:-:b](%d) <%s>: [-:-:-]\n%s\n",
 		"\n\n", len(chatBody.Messages), cfg.ToolRole, toolMsg)
 	// Create tool response message with the proper tool_call_id
+	// Mark shell commands as always visible
+	isShellCommand := fc.Name == "execute_command"
 	toolResponseMsg := models.RoleMsg{
-		Role:       cfg.ToolRole,
+		Role:           cfg.ToolRole,
-		Content:    toolMsg,
+		Content:        toolMsg,
-		ToolCallID: lastToolCall.ID, // Use the stored tool call ID
+		ToolCallID:     lastToolCall.ID,
+		IsShellCommand: isShellCommand,
 	}
 	chatBody.Messages = append(chatBody.Messages, toolResponseMsg)
 	logger.Debug("findCall: added actual tool response", "role", toolResponseMsg.Role, "content_len", len(toolResponseMsg.Content), "tool_call_id", toolResponseMsg.ToolCallID, "message_count_after_add", len(chatBody.Messages))
@@ -1195,12 +1200,42 @@ func findCall(msg, toolCall string) bool {
 
 func chatToTextSlice(messages []models.RoleMsg, showSys bool) []string {
 	resp := make([]string, len(messages))
-	for i, msg := range messages {
+	for i := range messages {
-		// INFO: skips system msg and tool msg
+		icon := fmt.Sprintf("[-:-:b](%d) <%s>:[-:-:-]", i, messages[i].Role)
-		if !showSys && (msg.Role == cfg.ToolRole || msg.Role == "system") {
+		// Handle tool call indicators (assistant messages with tool call but empty content)
+		if messages[i].Role == cfg.AssistantRole && messages[i].ToolCall != nil && messages[i].ToolCall.ID != "" {
+			// This is a tool call indicator - show collapsed
+			if toolCollapsed {
+				toolName := messages[i].ToolCall.Name
+				resp[i] = fmt.Sprintf("%s\n[yellow::i][tool call: %s (press Ctrl+T to expand)][-:-:-]\n", icon, toolName)
+			} else {
+				// Show full tool call info
+				toolName := messages[i].ToolCall.Name
+				resp[i] = fmt.Sprintf("%s\n%s\n[yellow::i][tool call: %s][-:-:-]\nargs: %s\nid: %s\n", icon, messages[i].GetText(), toolName, messages[i].ToolCall.Args, messages[i].ToolCall.ID)
+			}
 			continue
 		}
-		resp[i] = msg.ToText(i)
+		// Handle tool responses
+		if messages[i].Role == cfg.ToolRole || messages[i].Role == "tool" {
+			// Always show shell commands
+			if messages[i].IsShellCommand {
+				resp[i] = MsgToText(i, &messages[i])
+				continue
+			}
+			// Hide non-shell tool responses when collapsed
+			if toolCollapsed {
+				resp[i] = icon + "\n[yellow::i][tool resp (press Ctrl+T to expand)][-:-:-]\n"
+				continue
+			}
+			// When expanded, show tool responses
+			resp[i] = MsgToText(i, &messages[i])
+			continue
+		}
+		// INFO: skips system msg when showSys is false
+		if !showSys && messages[i].Role == "system" {
+			continue
+		}
+		resp[i] = MsgToText(i, &messages[i])
 	}
 	return resp
 }
@@ -1208,7 +1243,6 @@ func chatToTextSlice(messages []models.RoleMsg, showSys bool) []string {
 func chatToText(messages []models.RoleMsg, showSys bool) string {
 	s := chatToTextSlice(messages, showSys)
 	text := strings.Join(s, "\n")
-
 	// Collapse thinking blocks if enabled
 	if thinkingCollapsed {
 		text = thinkRE.ReplaceAllStringFunc(text, func(match string) string {
@@ -1232,27 +1266,9 @@ func chatToText(messages []models.RoleMsg, showSys bool) string {
 			}
 		}
 	}
-
 	return text
 }
 
-func removeThinking(chatBody *models.ChatBody) {
-	msgs := []models.RoleMsg{}
-	for _, msg := range chatBody.Messages {
-		// Filter out tool messages and thinking markers
-		if msg.Role == cfg.ToolRole {
-			continue
-		}
-		// find thinking and remove it
-		rm := models.RoleMsg{
-			Role:    msg.Role,
-			Content: thinkRE.ReplaceAllString(msg.Content, ""),
-		}
-		msgs = append(msgs, rm)
-	}
-	chatBody.Messages = msgs
-}
-
 func addNewChat(chatName string) {
 	id, err := store.ChatGetMaxID()
 	if err != nil {
@@ -1374,15 +1390,6 @@ func init() {
 		os.Exit(1)
 		return
 	}
-	// Set image base directory for path display
-	baseDir := cfg.FilePickerDir
-	if baseDir == "" || baseDir == "." {
-		// Resolve "." to current working directory
-		if wd, err := os.Getwd(); err == nil {
-			baseDir = wd
-		}
-	}
-	models.SetImageBaseDir(baseDir)
 	defaultStarter = []models.RoleMsg{
 		{Role: "system", Content: basicSysMsg},
 		{Role: cfg.AssistantRole, Content: defaultFirstMsg},
@@ -1397,8 +1404,6 @@ func init() {
 	}
 	// load cards
 	basicCard.Role = cfg.AssistantRole
-	// toolCard.Role = cfg.AssistantRole
-	//
 	logLevel.Set(slog.LevelInfo)
 	logger = slog.New(slog.NewTextHandler(logfile, &slog.HandlerOptions{Level: logLevel}))
 	store = storage.NewProviderSQL(cfg.DBPATH, logger)

bot_test.go

@@ -1,12 +1,10 @@
 package main
-
 import (
 	"gf-lt/config"
 	"gf-lt/models"
 	"reflect"
 	"testing"
 )
-
 func TestConsolidateConsecutiveAssistantMessages(t *testing.T) {
 	// Mock config for testing
 	testCfg := &config.Config{
@@ -14,7 +12,6 @@ func TestConsolidateConsecutiveAssistantMessages(t *testing.T) {
 		WriteNextMsgAsCompletionAgent: "",
 	}
 	cfg = testCfg
-
 	tests := []struct {
 		name     string
 		input    []models.RoleMsg
@@ -114,38 +111,31 @@ func TestConsolidateConsecutiveAssistantMessages(t *testing.T) {
 			},
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			result := consolidateAssistantMessages(tt.input)
-
 			if len(result) != len(tt.expected) {
 				t.Errorf("Expected %d messages, got %d", len(tt.expected), len(result))
 				t.Logf("Result: %+v", result)
 				t.Logf("Expected: %+v", tt.expected)
 				return
 			}
-
 			for i, expectedMsg := range tt.expected {
 				if i >= len(result) {
 					t.Errorf("Result has fewer messages than expected at index %d", i)
 					continue
 				}
-
 				actualMsg := result[i]
 				if actualMsg.Role != expectedMsg.Role {
 					t.Errorf("Message %d: expected role '%s', got '%s'", i, expectedMsg.Role, actualMsg.Role)
 				}
-
 				if actualMsg.Content != expectedMsg.Content {
 					t.Errorf("Message %d: expected content '%s', got '%s'", i, expectedMsg.Content, actualMsg.Content)
 				}
-
 				if actualMsg.ToolCallID != expectedMsg.ToolCallID {
 					t.Errorf("Message %d: expected ToolCallID '%s', got '%s'", i, expectedMsg.ToolCallID, actualMsg.ToolCallID)
 				}
 			}
-
 			// Additional check: ensure no messages were lost
 			if !reflect.DeepEqual(result, tt.expected) {
 				t.Errorf("Result does not match expected:\nResult:   %+v\nExpected: %+v", result, tt.expected)
@@ -153,7 +143,6 @@ func TestConsolidateConsecutiveAssistantMessages(t *testing.T) {
 		})
 	}
 }
-
 func TestUnmarshalFuncCall(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -213,7 +202,6 @@ func TestUnmarshalFuncCall(t *testing.T) {
 			wantErr: true,
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := unmarshalFuncCall(tt.jsonStr)
@@ -238,7 +226,6 @@ func TestUnmarshalFuncCall(t *testing.T) {
 		})
 	}
 }
-
 func TestConvertJSONToMapStringString(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -265,7 +252,6 @@ func TestConvertJSONToMapStringString(t *testing.T) {
 			wantErr: true,
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := convertJSONToMapStringString(tt.jsonStr)
@@ -287,7 +273,6 @@ func TestConvertJSONToMapStringString(t *testing.T) {
 		})
 	}
 }
-
 func TestParseKnownToTag(t *testing.T) {
 	tests := []struct {
 		name        string
@@ -378,7 +363,6 @@ func TestParseKnownToTag(t *testing.T) {
 			wantKnownTo: []string{"Alice", "Bob", "Carl"},
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			// Set up config
@@ -402,7 +386,6 @@ func TestParseKnownToTag(t *testing.T) {
 		})
 	}
 }
-
 func TestProcessMessageTag(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -498,7 +481,6 @@ func TestProcessMessageTag(t *testing.T) {
 			},
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			testCfg := &config.Config{
@@ -529,7 +511,6 @@ func TestProcessMessageTag(t *testing.T) {
 		})
 	}
 }
-
 func TestFilterMessagesForCharacter(t *testing.T) {
 	messages := []models.RoleMsg{
 		{Role: "system", Content: "System message", KnownTo: nil}, // visible to all
@@ -539,7 +520,6 @@ func TestFilterMessagesForCharacter(t *testing.T) {
 		{Role: "Alice", Content: "Private to Carl", KnownTo: []string{"Alice", "Carl"}},
 		{Role: "Carl", Content: "Hi all", KnownTo: nil}, // visible to all
 	}
-
 	tests := []struct {
 		name        string
 		enabled     bool
@@ -583,7 +563,6 @@ func TestFilterMessagesForCharacter(t *testing.T) {
 			wantIndices: []int{0, 1, 5},
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			testCfg := &config.Config{
@@ -591,15 +570,12 @@ func TestFilterMessagesForCharacter(t *testing.T) {
 				CharSpecificContextTag:     "@",
 			}
 			cfg = testCfg
-
 			got := filterMessagesForCharacter(messages, tt.character)
-
 			if len(got) != len(tt.wantIndices) {
 				t.Errorf("filterMessagesForCharacter() returned %d messages, want %d", len(got), len(tt.wantIndices))
 				t.Logf("got: %v", got)
 				return
 			}
-
 			for i, idx := range tt.wantIndices {
 				if got[i].Content != messages[idx].Content {
 					t.Errorf("filterMessagesForCharacter() message %d content = %q, want %q", i, got[i].Content, messages[idx].Content)
@@ -608,7 +584,6 @@ func TestFilterMessagesForCharacter(t *testing.T) {
 		})
 	}
 }
-
 func TestRoleMsgCopyPreservesKnownTo(t *testing.T) {
 	// Test that the Copy() method preserves the KnownTo field
 	originalMsg := models.RoleMsg{
@@ -616,9 +591,7 @@ func TestRoleMsgCopyPreservesKnownTo(t *testing.T) {
 		Content: "Test message",
 		KnownTo: []string{"Bob", "Charlie"},
 	}
-
 	copiedMsg := originalMsg.Copy()
-
 	if copiedMsg.Role != originalMsg.Role {
 		t.Errorf("Copy() failed to preserve Role: got %q, want %q", copiedMsg.Role, originalMsg.Role)
 	}
@@ -635,7 +608,6 @@ func TestRoleMsgCopyPreservesKnownTo(t *testing.T) {
 		t.Errorf("Copy() failed to preserve hasContentParts flag")
 	}
 }
-
 func TestKnownToFieldPreservationScenario(t *testing.T) {
 	// Test the specific scenario from the log where KnownTo field was getting lost
 	originalMsg := models.RoleMsg{
@@ -643,28 +615,22 @@ func TestKnownToFieldPreservationScenario(t *testing.T) {
 		Content: `Alice: "Okay, Bob. The word is... **'Ephemeral'**. (ooc: @Bob@)"`,
 		KnownTo: []string{"Bob"}, // This was detected in the log
 	}
-
 	t.Logf("Original message - Role: %s, Content: %s, KnownTo: %v",
 		originalMsg.Role, originalMsg.Content, originalMsg.KnownTo)
-
 	// Simulate what happens when the message gets copied during processing
 	copiedMsg := originalMsg.Copy()
-
 	t.Logf("Copied message - Role: %s, Content: %s, KnownTo: %v",
 		copiedMsg.Role, copiedMsg.Content, copiedMsg.KnownTo)
-
 	// Check if KnownTo field survived the copy
 	if len(copiedMsg.KnownTo) == 0 {
 		t.Error("ERROR: KnownTo field was lost during copy!")
 	} else {
 		t.Log("SUCCESS: KnownTo field was preserved during copy!")
 	}
-
 	// Verify the content is the same
 	if copiedMsg.Content != originalMsg.Content {
 		t.Errorf("Content was changed during copy: got %s, want %s", copiedMsg.Content, originalMsg.Content)
 	}
-
 	// Verify the KnownTo slice is properly copied
 	if !reflect.DeepEqual(copiedMsg.KnownTo, originalMsg.KnownTo) {
 		t.Errorf("KnownTo was not properly copied: got %v, want %v", copiedMsg.KnownTo, originalMsg.KnownTo)

config.example.toml

@@ -12,7 +12,7 @@ OpenRouterChatAPI = "https://openrouter.ai/api/v1/chat/completions"
 # OpenRouterToken = ""
 # embeddings
 EmbedURL = "http://localhost:8082/v1/embeddings"
-HFToken = false
+HFToken = ""
 #
 ShowSys = true
 LogFile = "log.txt"
@@ -27,7 +27,6 @@ AutoCleanToolCallsFromCtx = false
 RAGEnabled = false
 RAGBatchSize = 1
 RAGWordLimit = 80
-RAGWorkers = 2
 RAGDir = "ragimport"
 # extra tts
 TTS_ENABLED = false

config/config.go

@@ -18,7 +18,6 @@ type Config struct {
 	UserRole                      string `toml:"UserRole"`
 	ToolRole                      string `toml:"ToolRole"`
 	ToolUse                       bool   `toml:"ToolUse"`
-	ThinkUse                      bool   `toml:"ThinkUse"`
 	StripThinkingFromAPI          bool   `toml:"StripThinkingFromAPI"`
 	ReasoningEffort               string `toml:"ReasoningEffort"`
 	AssistantRole                 string `toml:"AssistantRole"`
@@ -40,7 +39,6 @@ type Config struct {
 	// rag settings
 	RAGEnabled   bool   `toml:"RAGEnabled"`
 	RAGDir       string `toml:"RAGDir"`
-	RAGWorkers   uint32 `toml:"RAGWorkers"`
 	RAGBatchSize int    `toml:"RAGBatchSize"`
 	RAGWordLimit uint32 `toml:"RAGWordLimit"`
 	// deepseek
@@ -125,6 +123,9 @@ func LoadConfig(fn string) (*Config, error) {
 	if config.CompletionAPI != "" {
 		config.ApiLinks = append(config.ApiLinks, config.CompletionAPI)
 	}
+	if config.RAGDir == "" {
+		config.RAGDir = "ragimport"
+	}
 	// if any value is empty fill with default
 	return config, nil
 }

docs/config.md

@@ -80,9 +80,6 @@ This document explains how to set up and configure the application using the `co
 #### RAGWordLimit (`80`)
 - Maximum number of words in a batch to tokenize and store.
 
-#### RAGWorkers (`2`)
-- Number of concurrent workers for RAG processing.
-
 #### RAGDir (`"ragimport"`)
 - Directory containing documents for RAG processing.
 
@@ -165,9 +162,6 @@ Those could be switched in program, but also bould be setup in config.
 #### ToolUse
 - Enable or disable explanation of tools to llm, so it could use them.
 
-#### ThinkUse
-- Enable or disable insertion of  JsonSerializerToken at the beggining of llm resp.
-
 ### StripThinkingFromAPI (`true`)
 - Strip thinking blocks from messages before sending to LLM. Keeps them in chat history for local viewing but reduces token usage in API calls.
 

go.mod

@@ -6,17 +6,19 @@ require (
 	github.com/BurntSushi/toml v1.5.0
 	github.com/GrailFinder/google-translate-tts v0.1.3
 	github.com/GrailFinder/searchagent v0.2.0
+	github.com/PuerkitoBio/goquery v1.11.0
 	github.com/gdamore/tcell/v2 v2.13.2
 	github.com/glebarez/go-sqlite v1.22.0
 	github.com/gopxl/beep/v2 v2.1.1
 	github.com/gordonklaus/portaudio v0.0.0-20250206071425-98a94950218b
 	github.com/jmoiron/sqlx v1.4.0
+	github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728
 	github.com/neurosnap/sentences v1.1.2
 	github.com/rivo/tview v0.42.0
+	github.com/yuin/goldmark v1.4.13
 )
 
 require (
-	github.com/PuerkitoBio/goquery v1.11.0 // indirect
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/ebitengine/oto/v3 v3.4.0 // indirect

go.sum

@@ -43,6 +43,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
 github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
+github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+ExRDqGQltzXqN/xypdKP86niVn8=
+github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
@@ -67,6 +69,7 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=

helpfuncs.go

@@ -15,8 +15,6 @@ import (
 	"time"
 	"unicode"
 
-	"math/rand/v2"
-
 	"github.com/rivo/tview"
 )
 
@@ -32,7 +30,6 @@ func startModelColorUpdater() {
 
 		// Initial check
 		updateCachedModelColor()
-
 		for range ticker.C {
 			updateCachedModelColor()
 		}
@@ -69,21 +66,30 @@ func isASCII(s string) bool {
 	return true
 }
 
+func mapToString[V any](m map[string]V) string {
+	rs := strings.Builder{}
+	for k, v := range m {
+		fmt.Fprintf(&rs, "%v: %v\n", k, v)
+	}
+	return rs.String()
+}
+
 // stripThinkingFromMsg removes thinking blocks from assistant messages.
 // Skips user, tool, and system messages as they may contain thinking examples.
 func stripThinkingFromMsg(msg *models.RoleMsg) *models.RoleMsg {
 	if !cfg.StripThinkingFromAPI {
 		return msg
 	}
-	// Skip user, tool, and system messages - they might contain thinking examples
+	// Skip user, tool, they might contain thinking and system messages - examples
 	if msg.Role == cfg.UserRole || msg.Role == cfg.ToolRole || msg.Role == "system" {
 		return msg
 	}
 	// Strip thinking from assistant messages
-	if thinkRE.MatchString(msg.Content) {
+	msgText := msg.GetText()
-		msg.Content = thinkRE.ReplaceAllString(msg.Content, "")
+	if thinkRE.MatchString(msgText) {
-		// Clean up any double newlines that might result
+		cleanedText := thinkRE.ReplaceAllString(msgText, "")
-		msg.Content = strings.TrimSpace(msg.Content)
+		cleanedText = strings.TrimSpace(cleanedText)
+		msg.SetText(cleanedText)
 	}
 	return msg
 }
@@ -211,8 +217,10 @@ func startNewChat(keepSysP bool) {
 	chatBody.Messages = chatBody.Messages[:2]
 	textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
 	newChat := &models.Chat{
-		ID:   id + 1,
+		ID:        id + 1,
-		Name: fmt.Sprintf("%d_%s", id+1, cfg.AssistantRole),
+		Name:      fmt.Sprintf("%d_%s", id+1, cfg.AssistantRole),
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
 		// chat is written to db when we get first llm response (or any)
 		// actual chat history (messages) would be parsed then
 		Msgs:  "",
@@ -357,7 +365,7 @@ func makeStatusLine() string {
 	}
 	// Get model color based on load status for local llama.cpp models
 	modelColor := getModelColor()
-	statusLine := fmt.Sprintf(statusLineTempl, boolColors[botRespMode], activeChatName,
+	statusLine := fmt.Sprintf(statusLineTempl, activeChatName,
 		boolColors[cfg.ToolUse], modelColor, chatBody.Model, boolColors[cfg.SkipLLMResp],
 		cfg.CurrentAPI, persona, botPersona)
 	if cfg.STT_ENABLED {
@@ -373,16 +381,6 @@ func makeStatusLine() string {
 	return statusLine + imageInfo + shellModeInfo
 }
 
-var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
-
-func randString(n int) string {
-	b := make([]rune, n)
-	for i := range b {
-		b[i] = letters[rand.IntN(len(letters))]
-	}
-	return string(b)
-}
-
 // set of roles within card definition and mention in chat history
 func listChatRoles() []string {
 	currentChat, ok := chatMap[activeChatName]
@@ -426,12 +424,11 @@ func deepseekModelValidator() error {
 
 func toggleShellMode() {
 	shellMode = !shellMode
+	setShellMode(shellMode)
 	if shellMode {
-		// Update input placeholder to indicate shell mode
+		shellInput.SetLabel(fmt.Sprintf("[%s]$ ", cfg.FilePickerDir))
-		textArea.SetPlaceholder("SHELL MODE: Enter command and press <Esc> to execute")
 	} else {
-		// Reset to normal mode
+		textArea.SetPlaceholder("input is multiline; press <Enter> to start the next line;\npress <Esc> to send the message.")
-		textArea.SetPlaceholder("input is multiline; press <Enter> to start the next line;\npress <Esc> to send the message. Alt+1 to exit shell mode")
 	}
 	updateStatusLine()
 }
@@ -443,23 +440,29 @@ func updateFlexLayout() {
 	}
 	flex.Clear()
 	flex.AddItem(textView, 0, 40, false)
-	flex.AddItem(textArea, 0, 10, false)
+	if shellMode {
+		flex.AddItem(shellInput, 0, 10, false)
+	} else {
+		flex.AddItem(textArea, 0, 10, false)
+	}
 	if positionVisible {
 		flex.AddItem(statusLineWidget, 0, 2, false)
 	}
 	// Keep focus on currently focused widget
 	focused := app.GetFocus()
-	if focused == textView {
+	switch {
+	case focused == textView:
 		app.SetFocus(textView)
-	} else {
+	case shellMode:
+		app.SetFocus(shellInput)
+	default:
 		app.SetFocus(textArea)
 	}
 }
 
 func executeCommandAndDisplay(cmdText string) {
-	// Parse the command (split by spaces, but handle quoted arguments)
+	cmdText = strings.TrimSpace(cmdText)
-	cmdParts := parseCommand(cmdText)
+	if cmdText == "" {
-	if len(cmdParts) == 0 {
 		fmt.Fprintf(textView, "\n[red]Error: No command provided[-:-:-]\n")
 		if scrollToEndEnabled {
 			textView.ScrollToEnd()
@@ -467,17 +470,63 @@ func executeCommandAndDisplay(cmdText string) {
 		colorText()
 		return
 	}
-	command := cmdParts[0]
+	workingDir := cfg.FilePickerDir
-	args := []string{}
+	// Handle cd command specially to update working directory
-	if len(cmdParts) > 1 {
+	if strings.HasPrefix(cmdText, "cd ") {
-		args = cmdParts[1:]
+		newDir := strings.TrimPrefix(cmdText, "cd ")
+		newDir = strings.TrimSpace(newDir)
+		// Handle cd ~ or cdHOME
+		if strings.HasPrefix(newDir, "~") {
+			home := os.Getenv("HOME")
+			newDir = strings.Replace(newDir, "~", home, 1)
+		}
+		// Check if directory exists
+		if _, err := os.Stat(newDir); err == nil {
+			workingDir = newDir
+			cfg.FilePickerDir = workingDir
+			// Update shell input label with new directory
+			shellInput.SetLabel(fmt.Sprintf("[%s]$ ", cfg.FilePickerDir))
+			outputContent := workingDir
+			// Add the command being executed to the chat
+			fmt.Fprintf(textView, "\n[-:-:b](%d) <%s>: [-:-:-]\n$ %s\n",
+				len(chatBody.Messages), cfg.ToolRole, cmdText)
+			fmt.Fprintf(textView, "%s\n", outputContent)
+			combinedMsg := models.RoleMsg{
+				Role:    cfg.ToolRole,
+				Content: "$ " + cmdText + "\n\n" + outputContent,
+			}
+			chatBody.Messages = append(chatBody.Messages, combinedMsg)
+			if scrollToEndEnabled {
+				textView.ScrollToEnd()
+			}
+			colorText()
+			return
+		} else {
+			outputContent := "cd: " + newDir + ": No such file or directory"
+			fmt.Fprintf(textView, "\n[-:-:b](%d) <%s>: [-:-:-]\n$ %s\n",
+				len(chatBody.Messages), cfg.ToolRole, cmdText)
+			fmt.Fprintf(textView, "[red]%s[-:-:-]\n", outputContent)
+			combinedMsg := models.RoleMsg{
+				Role:    cfg.ToolRole,
+				Content: "$ " + cmdText + "\n\n" + outputContent,
+			}
+			chatBody.Messages = append(chatBody.Messages, combinedMsg)
+			if scrollToEndEnabled {
+				textView.ScrollToEnd()
+			}
+			colorText()
+			return
+		}
 	}
-	// Create the command execution
+
-	cmd := exec.Command(command, args...)
+	// Use /bin/sh to support pipes, redirects, etc.
+	cmd := exec.Command("/bin/sh", "-c", cmdText)
+	cmd.Dir = workingDir
 	// Execute the command and get output
 	output, err := cmd.CombinedOutput()
 	// Add the command being executed to the chat
-	fmt.Fprintf(textView, "\n[yellow]$ %s[-:-:-]\n", cmdText)
+	fmt.Fprintf(textView, "\n[-:-:b](%d) <%s>: [-:-:-]\n$ %s\n",
+		len(chatBody.Messages), cfg.ToolRole, cmdText)
 	var outputContent string
 	if err != nil {
 		// Include both output and error
@@ -514,42 +563,11 @@ func executeCommandAndDisplay(cmdText string) {
 		textView.ScrollToEnd()
 	}
 	colorText()
-}
+	// Add command to history (avoid duplicates at the end)
-
+	if len(shellHistory) == 0 || shellHistory[len(shellHistory)-1] != cmdText {
-// parseCommand splits command string handling quotes properly
+		shellHistory = append(shellHistory, cmdText)
-func parseCommand(cmd string) []string {
-	var args []string
-	var current string
-	var inQuotes bool
-	var quoteChar rune
-	for _, r := range cmd {
-		switch r {
-		case '"', '\'':
-			if inQuotes {
-				if r == quoteChar {
-					inQuotes = false
-				} else {
-					current += string(r)
-				}
-			} else {
-				inQuotes = true
-				quoteChar = r
-			}
-		case ' ', '\t':
-			if inQuotes {
-				current += string(r)
-			} else if current != "" {
-				args = append(args, current)
-				current = ""
-			}
-		default:
-			current += string(r)
-		}
 	}
-	if current != "" {
+	shellHistoryPos = -1
-		args = append(args, current)
-	}
-	return args
 }
 
 // == search ==
@@ -791,3 +809,91 @@ func scanFiles(dir, filter string) []string {
 	scanRecursive(dir, 0, "")
 	return files
 }
+
+// models logic that is too complex for models package
+func MsgToText(i int, m *models.RoleMsg) string {
+	var contentStr string
+	var imageIndicators []string
+	if !m.HasContentParts {
+		contentStr = m.Content
+	} else {
+		var textParts []string
+		for _, part := range m.ContentParts {
+			switch p := part.(type) {
+			case models.TextContentPart:
+				if p.Type == "text" {
+					textParts = append(textParts, p.Text)
+				}
+			case models.ImageContentPart:
+				displayPath := p.Path
+				if displayPath == "" {
+					displayPath = "image"
+				} else {
+					displayPath = extractDisplayPath(displayPath, cfg.FilePickerDir)
+				}
+				imageIndicators = append(imageIndicators, fmt.Sprintf("[orange::i][image: %s][-:-:-]", displayPath))
+			case map[string]any:
+				if partType, exists := p["type"]; exists {
+					switch partType {
+					case "text":
+						if textVal, textExists := p["text"]; textExists {
+							if textStr, isStr := textVal.(string); isStr {
+								textParts = append(textParts, textStr)
+							}
+						}
+					case "image_url":
+						var displayPath string
+						if pathVal, pathExists := p["path"]; pathExists {
+							if pathStr, isStr := pathVal.(string); isStr && pathStr != "" {
+								displayPath = extractDisplayPath(pathStr, cfg.FilePickerDir)
+							}
+						}
+						if displayPath == "" {
+							displayPath = "image"
+						}
+						imageIndicators = append(imageIndicators, fmt.Sprintf("[orange::i][image: %s][-:-:-]", displayPath))
+					}
+				}
+			}
+		}
+		contentStr = strings.Join(textParts, " ") + " "
+	}
+	contentStr, _ = strings.CutPrefix(contentStr, m.Role+":")
+	icon := fmt.Sprintf("(%d) <%s>: ", i, m.Role)
+	var finalContent strings.Builder
+	if len(imageIndicators) > 0 {
+		for _, indicator := range imageIndicators {
+			finalContent.WriteString(indicator)
+			finalContent.WriteString("\n")
+		}
+	}
+	finalContent.WriteString(contentStr)
+	if m.Stats != nil {
+		fmt.Fprintf(&finalContent, "\n[gray::i][%d tok, %.1fs, %.1f t/s][-:-:-]", m.Stats.Tokens, m.Stats.Duration, m.Stats.TokensPerSec)
+	}
+	textMsg := fmt.Sprintf("[-:-:b]%s[-:-:-]\n%s\n", icon, finalContent.String())
+	return strings.ReplaceAll(textMsg, "\n\n", "\n")
+}
+
+// extractDisplayPath returns a path suitable for display, potentially relative to imageBaseDir
+func extractDisplayPath(p, bp string) string {
+	if p == "" {
+		return ""
+	}
+	// If base directory is set, try to make path relative to it
+	if bp != "" {
+		if rel, err := filepath.Rel(bp, p); err == nil {
+			// Check if relative path doesn't start with ".." (meaning it's within base dir)
+			// If it starts with "..", we might still want to show it as relative
+			// but for now we show full path if it goes outside base dir
+			if !strings.HasPrefix(rel, "..") {
+				p = rel
+			}
+		}
+	}
+	// Truncate long paths to last 60 characters if needed
+	if len(p) > 60 {
+		return "..." + p[len(p)-60:]
+	}
+	return p
+}

llm.go

@@ -11,12 +11,11 @@ import (
 
 var imageAttachmentPath string // Global variable to track image attachment for next message
 var lastImg string             // for ctrl+j
-var RAGMsg = "Retrieved context for user's query:\n"
 
 // containsToolSysMsg checks if the toolSysMsg already exists in the chat body
 func containsToolSysMsg() bool {
-	for _, msg := range chatBody.Messages {
+	for i := range chatBody.Messages {
-		if msg.Role == cfg.ToolRole && msg.Content == toolSysMsg {
+		if chatBody.Messages[i].Role == cfg.ToolRole && chatBody.Messages[i].Content == toolSysMsg {
 			return true
 		}
 	}
@@ -142,30 +141,14 @@ func (lcp LCPCompletion) FormMsg(msg, role string, resume bool) (io.Reader, erro
 		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	// if rag - add as system message to avoid conflicts with tool usage
-	if !resume && cfg.RAGEnabled {
-		um := chatBody.Messages[len(chatBody.Messages)-1].Content
-		logger.Debug("RAG is enabled, preparing RAG context", "user_message", um)
-		ragResp, err := chatRagUse(um)
-		if err != nil {
-			logger.Error("failed to form a rag msg", "error", err)
-			return nil, err
-		}
-		logger.Debug("RAG response received", "response_len", len(ragResp),
-			"response_preview", ragResp[:min(len(ragResp), 100)])
-		// Use system role for RAG context to avoid conflicts with tool usage
-		ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-		chatBody.Messages = append(chatBody.Messages, ragMsg)
-		logger.Debug("RAG message added to chat body", "message_count", len(chatBody.Messages))
-	}
 	// sending description of the tools and how to use them
 	if cfg.ToolUse && !resume && role == cfg.UserRole && !containsToolSysMsg() {
 		chatBody.Messages = append(chatBody.Messages, models.RoleMsg{Role: cfg.ToolRole, Content: toolSysMsg})
 	}
 	filteredMessages, botPersona := filterMessagesForCurrentCharacter(chatBody.Messages)
 	messages := make([]string, len(filteredMessages))
-	for i, m := range filteredMessages {
+	for i := range filteredMessages {
-		messages[i] = stripThinkingFromMsg(&m).ToPrompt()
+		messages[i] = stripThinkingFromMsg(&filteredMessages[i]).ToPrompt()
 	}
 	prompt := strings.Join(messages, "\n")
 	// Add multimodal media markers to the prompt text when multimodal data is present
@@ -184,9 +167,6 @@ func (lcp LCPCompletion) FormMsg(msg, role string, resume bool) (io.Reader, erro
 		botMsgStart := "\n" + botPersona + ":\n"
 		prompt += botMsgStart
 	}
-	if cfg.ThinkUse && !cfg.ToolUse {
-		prompt += "<think>"
-	}
 	logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
 		"msg", msg, "resume", resume, "prompt", prompt, "multimodal_data_count", len(multimodalData))
 	payload := models.NewLCPReq(prompt, chatBody.Model, multimodalData,
@@ -236,13 +216,11 @@ func (op LCPChat) ParseChunk(data []byte) (*models.TextChunk, error) {
 		logger.Warn("LCPChat ParseChunk: no choices in response", "data", string(data))
 		return &models.TextChunk{Finished: true}, nil
 	}
-
 	lastChoice := llmchunk.Choices[len(llmchunk.Choices)-1]
 	resp := &models.TextChunk{
 		Chunk:     lastChoice.Delta.Content,
 		Reasoning: lastChoice.Delta.ReasoningContent,
 	}
-
 	// Check for tool calls in all choices, not just the last one
 	for _, choice := range llmchunk.Choices {
 		if len(choice.Delta.ToolCalls) > 0 {
@@ -257,7 +235,6 @@ func (op LCPChat) ParseChunk(data []byte) (*models.TextChunk, error) {
 			break // Process only the first tool call
 		}
 	}
-
 	if lastChoice.FinishReason == "stop" {
 		if resp.Chunk != "" {
 			logger.Error("text inside of finish llmchunk", "chunk", llmchunk)
@@ -304,23 +281,6 @@ func (op LCPChat) FormMsg(msg, role string, resume bool) (io.Reader, error) {
 		logger.Debug("LCPChat FormMsg: added message to chatBody", "role", newMsg.Role,
 			"content_len", len(newMsg.Content), "message_count_after_add", len(chatBody.Messages))
 	}
-	// if rag - add as system message to avoid conflicts with tool usage
-	if !resume && cfg.RAGEnabled {
-		um := chatBody.Messages[len(chatBody.Messages)-1].Content
-		logger.Debug("LCPChat: RAG is enabled, preparing RAG context", "user_message", um)
-		ragResp, err := chatRagUse(um)
-		if err != nil {
-			logger.Error("LCPChat: failed to form a rag msg", "error", err)
-			return nil, err
-		}
-		logger.Debug("LCPChat: RAG response received",
-			"response_len", len(ragResp), "response_preview", ragResp[:min(len(ragResp), 100)])
-		// Use system role for RAG context to avoid conflicts with tool usage
-		ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-		chatBody.Messages = append(chatBody.Messages, ragMsg)
-		logger.Debug("LCPChat: RAG message added to chat body", "role", ragMsg.Role,
-			"rag_content_len", len(ragMsg.Content), "message_count_after_rag", len(chatBody.Messages))
-	}
 	filteredMessages, _ := filterMessagesForCurrentCharacter(chatBody.Messages)
 	// openai /v1/chat does not support custom roles; needs to be user, assistant, system
 	// Add persona suffix to the last user message to indicate who the assistant should reply as
@@ -329,14 +289,23 @@ func (op LCPChat) FormMsg(msg, role string, resume bool) (io.Reader, error) {
 		Model:    chatBody.Model,
 		Stream:   chatBody.Stream,
 	}
-	for i, msg := range filteredMessages {
+	for i := range filteredMessages {
-		strippedMsg := *stripThinkingFromMsg(&msg)
+		strippedMsg := *stripThinkingFromMsg(&filteredMessages[i])
-		if strippedMsg.Role == cfg.UserRole {
+		switch strippedMsg.Role {
+		case cfg.UserRole:
 			bodyCopy.Messages[i] = strippedMsg
 			bodyCopy.Messages[i].Role = "user"
-		} else {
+		case cfg.AssistantRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "assistant"
+		case cfg.ToolRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "tool"
+		default:
 			bodyCopy.Messages[i] = strippedMsg
 		}
+		// Clear ToolCalls - they're stored in chat history for display but not sent to LLM
+		// bodyCopy.Messages[i].ToolCall = nil
 	}
 	// Clean null/empty messages to prevent API issues
 	bodyCopy.Messages = consolidateAssistantMessages(bodyCopy.Messages)
@@ -392,30 +361,14 @@ func (ds DeepSeekerCompletion) FormMsg(msg, role string, resume bool) (io.Reader
 		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	// if rag - add as system message to avoid conflicts with tool usage
-	if !resume && cfg.RAGEnabled {
-		um := chatBody.Messages[len(chatBody.Messages)-1].Content
-		logger.Debug("DeepSeekerCompletion: RAG is enabled, preparing RAG context", "user_message", um)
-		ragResp, err := chatRagUse(um)
-		if err != nil {
-			logger.Error("DeepSeekerCompletion: failed to form a rag msg", "error", err)
-			return nil, err
-		}
-		logger.Debug("DeepSeekerCompletion: RAG response received",
-			"response_len", len(ragResp), "response_preview", ragResp[:min(len(ragResp), 100)])
-		// Use system role for RAG context to avoid conflicts with tool usage
-		ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-		chatBody.Messages = append(chatBody.Messages, ragMsg)
-		logger.Debug("DeepSeekerCompletion: RAG message added to chat body", "message_count", len(chatBody.Messages))
-	}
 	// sending description of the tools and how to use them
 	if cfg.ToolUse && !resume && role == cfg.UserRole && !containsToolSysMsg() {
 		chatBody.Messages = append(chatBody.Messages, models.RoleMsg{Role: cfg.ToolRole, Content: toolSysMsg})
 	}
 	filteredMessages, botPersona := filterMessagesForCurrentCharacter(chatBody.Messages)
 	messages := make([]string, len(filteredMessages))
-	for i, m := range filteredMessages {
+	for i := range filteredMessages {
-		messages[i] = stripThinkingFromMsg(&m).ToPrompt()
+		messages[i] = stripThinkingFromMsg(&filteredMessages[i]).ToPrompt()
 	}
 	prompt := strings.Join(messages, "\n")
 	// strings builder?
@@ -423,9 +376,6 @@ func (ds DeepSeekerCompletion) FormMsg(msg, role string, resume bool) (io.Reader
 		botMsgStart := "\n" + botPersona + ":\n"
 		prompt += botMsgStart
 	}
-	if cfg.ThinkUse && !cfg.ToolUse {
-		prompt += "<think>"
-	}
 	logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
 		"msg", msg, "resume", resume, "prompt", prompt)
 	payload := models.NewDSCompletionReq(prompt, chatBody.Model,
@@ -480,22 +430,6 @@ func (ds DeepSeekerChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
 		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	// if rag - add as system message to avoid conflicts with tool usage
-	if !resume && cfg.RAGEnabled {
-		um := chatBody.Messages[len(chatBody.Messages)-1].Content
-		logger.Debug("RAG is enabled, preparing RAG context", "user_message", um)
-		ragResp, err := chatRagUse(um)
-		if err != nil {
-			logger.Error("failed to form a rag msg", "error", err)
-			return nil, err
-		}
-		logger.Debug("RAG response received", "response_len", len(ragResp),
-			"response_preview", ragResp[:min(len(ragResp), 100)])
-		// Use system role for RAG context to avoid conflicts with tool usage
-		ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-		chatBody.Messages = append(chatBody.Messages, ragMsg)
-		logger.Debug("RAG message added to chat body", "message_count", len(chatBody.Messages))
-	}
 	// Create copy of chat body with standardized user role
 	filteredMessages, _ := filterMessagesForCurrentCharacter(chatBody.Messages)
 	// Add persona suffix to the last user message to indicate who the assistant should reply as
@@ -504,14 +438,27 @@ func (ds DeepSeekerChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
 		Model:    chatBody.Model,
 		Stream:   chatBody.Stream,
 	}
-	for i, msg := range filteredMessages {
+	for i := range filteredMessages {
-		strippedMsg := *stripThinkingFromMsg(&msg)
+		strippedMsg := *stripThinkingFromMsg(&filteredMessages[i])
-		if strippedMsg.Role == cfg.UserRole || i == 1 {
+		switch strippedMsg.Role {
+		case cfg.UserRole:
+			if i == 1 {
+				bodyCopy.Messages[i] = strippedMsg
+				bodyCopy.Messages[i].Role = "user"
+			} else {
+				bodyCopy.Messages[i] = strippedMsg
+			}
+		case cfg.AssistantRole:
 			bodyCopy.Messages[i] = strippedMsg
-			bodyCopy.Messages[i].Role = "user"
+			bodyCopy.Messages[i].Role = "assistant"
-		} else {
+		case cfg.ToolRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "tool"
+		default:
 			bodyCopy.Messages[i] = strippedMsg
 		}
+		// Clear ToolCalls - they're stored in chat history for display but not sent to LLM
+		// bodyCopy.Messages[i].ToolCall = nil
 	}
 	// Clean null/empty messages to prevent API issues
 	bodyCopy.Messages = consolidateAssistantMessages(bodyCopy.Messages)
@@ -558,30 +505,14 @@ func (or OpenRouterCompletion) FormMsg(msg, role string, resume bool) (io.Reader
 		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	// if rag - add as system message to avoid conflicts with tool usage
-	if !resume && cfg.RAGEnabled {
-		um := chatBody.Messages[len(chatBody.Messages)-1].Content
-		logger.Debug("RAG is enabled, preparing RAG context", "user_message", um)
-		ragResp, err := chatRagUse(um)
-		if err != nil {
-			logger.Error("failed to form a rag msg", "error", err)
-			return nil, err
-		}
-		logger.Debug("RAG response received", "response_len",
-			len(ragResp), "response_preview", ragResp[:min(len(ragResp), 100)])
-		// Use system role for RAG context to avoid conflicts with tool usage
-		ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-		chatBody.Messages = append(chatBody.Messages, ragMsg)
-		logger.Debug("RAG message added to chat body", "message_count", len(chatBody.Messages))
-	}
 	// sending description of the tools and how to use them
 	if cfg.ToolUse && !resume && role == cfg.UserRole && !containsToolSysMsg() {
 		chatBody.Messages = append(chatBody.Messages, models.RoleMsg{Role: cfg.ToolRole, Content: toolSysMsg})
 	}
 	filteredMessages, botPersona := filterMessagesForCurrentCharacter(chatBody.Messages)
 	messages := make([]string, len(filteredMessages))
-	for i, m := range filteredMessages {
+	for i := range filteredMessages {
-		messages[i] = stripThinkingFromMsg(&m).ToPrompt()
+		messages[i] = stripThinkingFromMsg(&filteredMessages[i]).ToPrompt()
 	}
 	prompt := strings.Join(messages, "\n")
 	// strings builder?
@@ -589,9 +520,6 @@ func (or OpenRouterCompletion) FormMsg(msg, role string, resume bool) (io.Reader
 		botMsgStart := "\n" + botPersona + ":\n"
 		prompt += botMsgStart
 	}
-	if cfg.ThinkUse && !cfg.ToolUse {
-		prompt += "<think>"
-	}
 	stopSlice := chatBody.MakeStopSliceExcluding("", listChatRoles())
 	logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
 		"msg", msg, "resume", resume, "prompt", prompt, "stop_strings", stopSlice)
@@ -679,22 +607,6 @@ func (or OpenRouterChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
 		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	// if rag - add as system message to avoid conflicts with tool usage
-	if !resume && cfg.RAGEnabled {
-		um := chatBody.Messages[len(chatBody.Messages)-1].Content
-		logger.Debug("RAG is enabled, preparing RAG context", "user_message", um)
-		ragResp, err := chatRagUse(um)
-		if err != nil {
-			logger.Error("failed to form a rag msg", "error", err)
-			return nil, err
-		}
-		logger.Debug("RAG response received", "response_len", len(ragResp),
-			"response_preview", ragResp[:min(len(ragResp), 100)])
-		// Use system role for RAG context to avoid conflicts with tool usage
-		ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-		chatBody.Messages = append(chatBody.Messages, ragMsg)
-		logger.Debug("RAG message added to chat body", "message_count", len(chatBody.Messages))
-	}
 	// Create copy of chat body with standardized user role
 	filteredMessages, _ := filterMessagesForCurrentCharacter(chatBody.Messages)
 	// Add persona suffix to the last user message to indicate who the assistant should reply as
@@ -703,14 +615,24 @@ func (or OpenRouterChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
 		Model:    chatBody.Model,
 		Stream:   chatBody.Stream,
 	}
-	for i, msg := range filteredMessages {
+	for i := range filteredMessages {
-		strippedMsg := *stripThinkingFromMsg(&msg)
+		strippedMsg := *stripThinkingFromMsg(&filteredMessages[i])
-		bodyCopy.Messages[i] = strippedMsg
+		switch strippedMsg.Role {
-		// Standardize role if it's a user role
+		case cfg.UserRole:
-		if bodyCopy.Messages[i].Role == cfg.UserRole {
 			bodyCopy.Messages[i] = strippedMsg
 			bodyCopy.Messages[i].Role = "user"
+		case cfg.AssistantRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "assistant"
+		case cfg.ToolRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "tool"
+		default:
+			bodyCopy.Messages[i] = strippedMsg
 		}
+		// Clear ToolCalls - they're stored in chat history for display but not sent to LLM
+		// literally deletes data that we need
+		// bodyCopy.Messages[i].ToolCall = nil
 	}
 	// Clean null/empty messages to prevent API issues
 	bodyCopy.Messages = consolidateAssistantMessages(bodyCopy.Messages)

main.go

@@ -7,14 +7,18 @@ import (
 var (
 	boolColors        = map[bool]string{true: "green", false: "red"}
 	botRespMode       = false
+	toolRunningMode   = false
 	editMode          = false
 	roleEditMode      = false
 	injectRole        = true
 	selectedIndex     = int(-1)
 	shellMode         = false
-	thinkingCollapsed = false
+	shellHistory      []string
-	statusLineTempl   = "help (F12) | [%s:-:b]llm writes[-:-:-] (F6 to interrupt) | chat: [orange:-:b]%s[-:-:-] (F1) | [%s:-:b]tool use[-:-:-] (ctrl+k) | model: [%s:-:b]%s[-:-:-] (ctrl+l) | [%s:-:b]skip LLM resp[-:-:-] (F10)\nAPI: [orange:-:b]%s[-:-:-] (ctrl+v) | writing as: [orange:-:b]%s[-:-:-] (ctrl+q) | bot will write as [orange:-:b]%s[-:-:-] (ctrl+x)"
+	shellHistoryPos   int = -1
-	focusSwitcher     = map[tview.Primitive]tview.Primitive{}
+	thinkingCollapsed     = false
+	toolCollapsed         = true
+	statusLineTempl       = "help (F12) | chat: [orange:-:b]%s[-:-:-] (F1) | [%s:-:b]tool use[-:-:-] (ctrl+k) | model: [%s:-:b]%s[-:-:-] (ctrl+l) | [%s:-:b]skip LLM resp[-:-:-] (F10)\nAPI: [orange:-:b]%s[-:-:-] (ctrl+v) | writing as: [orange:-:b]%s[-:-:-] (ctrl+q) | bot will write as [orange:-:b]%s[-:-:-] (ctrl+x)"
+	focusSwitcher         = map[tview.Primitive]tview.Primitive{}
 )
 
 func main() {

main_test.go

@@ -1,42 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"gf-lt/config"
-	"gf-lt/models"
-	"strings"
-	"testing"
-)
-
-func TestRemoveThinking(t *testing.T) {
-	cases := []struct {
-		cb       *models.ChatBody
-		toolMsgs uint8
-	}{
-		{cb: &models.ChatBody{
-			Stream: true,
-			Messages: []models.RoleMsg{
-				{Role: "tool", Content: "should be ommited"},
-				{Role: "system", Content: "should stay"},
-				{Role: "user", Content: "hello, how are you?"},
-				{Role: "assistant", Content: "Oh, hi. <think>I should thank user and continue the conversation</think> I am geat, thank you! How are you?"},
-			},
-		},
-			toolMsgs: uint8(1),
-		},
-	}
-	for i, tc := range cases {
-					t.Run(fmt.Sprintf("run_%d", i), func(t *testing.T) {
-						cfg = &config.Config{ToolRole: "tool"} // Initialize cfg.ToolRole for test
-						mNum := len(tc.cb.Messages)
-						removeThinking(tc.cb)
-						if len(tc.cb.Messages) != mNum-int(tc.toolMsgs) {
-							t.Errorf("failed to delete tools msg %v; expected %d, got %d", tc.cb.Messages, mNum-int(tc.toolMsgs), len(tc.cb.Messages))
-						}
-						for _, msg := range tc.cb.Messages {
-							if strings.Contains(msg.Content, "<think>") {
-								t.Errorf("msg contains think tag; msg: %s\n", msg.Content)
-							}
-						}
-					})	}
-}

models/models.go

@@ -5,28 +5,21 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"path/filepath"
 	"strings"
 )
 
-var (
-	// imageBaseDir is the base directory for displaying image paths.
-	// If set, image paths will be shown relative to this directory.
-	imageBaseDir = ""
-)
-
-// SetImageBaseDir sets the base directory for displaying image paths.
-// If dir is empty, full paths will be shown.
-func SetImageBaseDir(dir string) {
-	imageBaseDir = dir
-}
-
 type FuncCall struct {
 	ID   string            `json:"id,omitempty"`
 	Name string            `json:"name"`
 	Args map[string]string `json:"args"`
 }
 
+type ToolCall struct {
+	ID   string `json:"id,omitempty"`
+	Name string `json:"name"`
+	Args string `json:"arguments"`
+}
+
 type LLMResp struct {
 	Choices []struct {
 		FinishReason string `json:"finish_reason"`
@@ -105,42 +98,59 @@ type ImageContentPart struct {
 
 // RoleMsg represents a message with content that can be either a simple string or structured content parts
 type RoleMsg struct {
-	Role            string   `json:"role"`
+	Role            string         `json:"role"`
-	Content         string   `json:"-"`
+	Content         string         `json:"-"`
-	ContentParts    []any    `json:"-"`
+	ContentParts    []any          `json:"-"`
-	ToolCallID      string   `json:"tool_call_id,omitempty"` // For tool response messages
+	ToolCallID      string         `json:"tool_call_id,omitempty"`     // For tool response messages
-	KnownTo         []string `json:"known_to,omitempty"`
+	ToolCall        *ToolCall      `json:"tool_call,omitempty"`        // For assistant messages with tool calls
-	hasContentParts bool     // Flag to indicate which content type to marshal
+	IsShellCommand  bool           `json:"is_shell_command,omitempty"` // True for shell command outputs (always shown)
+	KnownTo         []string       `json:"known_to,omitempty"`
+	Stats           *ResponseStats `json:"stats"`
+	HasContentParts bool           // Flag to indicate which content type to marshal
 }
 
 // MarshalJSON implements custom JSON marshaling for RoleMsg
-func (m *RoleMsg) MarshalJSON() ([]byte, error) {
+//
-	if m.hasContentParts {
+//nolint:gocritic
+func (m RoleMsg) MarshalJSON() ([]byte, error) {
+	if m.HasContentParts {
 		// Use structured content format
 		aux := struct {
-			Role       string   `json:"role"`
+			Role           string         `json:"role"`
-			Content    []any    `json:"content"`
+			Content        []any          `json:"content"`
-			ToolCallID string   `json:"tool_call_id,omitempty"`
+			ToolCallID     string         `json:"tool_call_id,omitempty"`
-			KnownTo    []string `json:"known_to,omitempty"`
+			ToolCall       *ToolCall      `json:"tool_call,omitempty"`
+			IsShellCommand bool           `json:"is_shell_command,omitempty"`
+			KnownTo        []string       `json:"known_to,omitempty"`
+			Stats          *ResponseStats `json:"stats,omitempty"`
 		}{
-			Role:       m.Role,
+			Role:           m.Role,
-			Content:    m.ContentParts,
+			Content:        m.ContentParts,
-			ToolCallID: m.ToolCallID,
+			ToolCallID:     m.ToolCallID,
-			KnownTo:    m.KnownTo,
+			ToolCall:       m.ToolCall,
+			IsShellCommand: m.IsShellCommand,
+			KnownTo:        m.KnownTo,
+			Stats:          m.Stats,
 		}
 		return json.Marshal(aux)
 	} else {
 		// Use simple content format
 		aux := struct {
-			Role       string   `json:"role"`
+			Role           string         `json:"role"`
-			Content    string   `json:"content"`
+			Content        string         `json:"content"`
-			ToolCallID string   `json:"tool_call_id,omitempty"`
+			ToolCallID     string         `json:"tool_call_id,omitempty"`
-			KnownTo    []string `json:"known_to,omitempty"`
+			ToolCall       *ToolCall      `json:"tool_call,omitempty"`
+			IsShellCommand bool           `json:"is_shell_command,omitempty"`
+			KnownTo        []string       `json:"known_to,omitempty"`
+			Stats          *ResponseStats `json:"stats,omitempty"`
 		}{
-			Role:       m.Role,
+			Role:           m.Role,
-			Content:    m.Content,
+			Content:        m.Content,
-			ToolCallID: m.ToolCallID,
+			ToolCallID:     m.ToolCallID,
-			KnownTo:    m.KnownTo,
+			ToolCall:       m.ToolCall,
+			IsShellCommand: m.IsShellCommand,
+			KnownTo:        m.KnownTo,
+			Stats:          m.Stats,
 		}
 		return json.Marshal(aux)
 	}
@@ -150,26 +160,35 @@ func (m *RoleMsg) MarshalJSON() ([]byte, error) {
 func (m *RoleMsg) UnmarshalJSON(data []byte) error {
 	// First, try to unmarshal as structured content format
 	var structured struct {
-		Role       string   `json:"role"`
+		Role           string         `json:"role"`
-		Content    []any    `json:"content"`
+		Content        []any          `json:"content"`
-		ToolCallID string   `json:"tool_call_id,omitempty"`
+		ToolCallID     string         `json:"tool_call_id,omitempty"`
-		KnownTo    []string `json:"known_to,omitempty"`
+		ToolCall       *ToolCall      `json:"tool_call,omitempty"`
+		IsShellCommand bool           `json:"is_shell_command,omitempty"`
+		KnownTo        []string       `json:"known_to,omitempty"`
+		Stats          *ResponseStats `json:"stats,omitempty"`
 	}
 	if err := json.Unmarshal(data, &structured); err == nil && len(structured.Content) > 0 {
 		m.Role = structured.Role
 		m.ContentParts = structured.Content
 		m.ToolCallID = structured.ToolCallID
+		m.ToolCall = structured.ToolCall
+		m.IsShellCommand = structured.IsShellCommand
 		m.KnownTo = structured.KnownTo
-		m.hasContentParts = true
+		m.Stats = structured.Stats
+		m.HasContentParts = true
 		return nil
 	}
 
 	// Otherwise, unmarshal as simple content format
 	var simple struct {
-		Role       string   `json:"role"`
+		Role           string         `json:"role"`
-		Content    string   `json:"content"`
+		Content        string         `json:"content"`
-		ToolCallID string   `json:"tool_call_id,omitempty"`
+		ToolCallID     string         `json:"tool_call_id,omitempty"`
-		KnownTo    []string `json:"known_to,omitempty"`
+		ToolCall       *ToolCall      `json:"tool_call,omitempty"`
+		IsShellCommand bool           `json:"is_shell_command,omitempty"`
+		KnownTo        []string       `json:"known_to,omitempty"`
+		Stats          *ResponseStats `json:"stats,omitempty"`
 	}
 	if err := json.Unmarshal(data, &simple); err != nil {
 		return err
@@ -177,86 +196,17 @@ func (m *RoleMsg) UnmarshalJSON(data []byte) error {
 	m.Role = simple.Role
 	m.Content = simple.Content
 	m.ToolCallID = simple.ToolCallID
+	m.ToolCall = simple.ToolCall
+	m.IsShellCommand = simple.IsShellCommand
 	m.KnownTo = simple.KnownTo
-	m.hasContentParts = false
+	m.Stats = simple.Stats
+	m.HasContentParts = false
 	return nil
 }
 
-func (m *RoleMsg) ToText(i int) string {
-	// Convert content to string representation
-	var contentStr string
-	var imageIndicators []string
-	if !m.hasContentParts {
-		contentStr = m.Content
-	} else {
-		// For structured content, collect text parts and image indicators
-		var textParts []string
-		for _, part := range m.ContentParts {
-			switch p := part.(type) {
-			case TextContentPart:
-				if p.Type == "text" {
-					textParts = append(textParts, p.Text)
-				}
-			case ImageContentPart:
-				// Collect image indicator
-				displayPath := p.Path
-				if displayPath == "" {
-					displayPath = "image"
-				} else {
-					displayPath = extractDisplayPath(displayPath)
-				}
-				imageIndicators = append(imageIndicators, fmt.Sprintf("[orange::i][image: %s][-:-:-]", displayPath))
-			case map[string]any:
-				if partType, exists := p["type"]; exists {
-					switch partType {
-					case "text":
-						if textVal, textExists := p["text"]; textExists {
-							if textStr, isStr := textVal.(string); isStr {
-								textParts = append(textParts, textStr)
-							}
-						}
-					case "image_url":
-						// Handle unmarshaled image content
-						var displayPath string
-						if pathVal, pathExists := p["path"]; pathExists {
-							if pathStr, isStr := pathVal.(string); isStr && pathStr != "" {
-								displayPath = extractDisplayPath(pathStr)
-							}
-						}
-						if displayPath == "" {
-							displayPath = "image"
-						}
-						imageIndicators = append(imageIndicators, fmt.Sprintf("[orange::i][image: %s][-:-:-]", displayPath))
-					}
-				}
-			}
-		}
-		contentStr = strings.Join(textParts, " ") + " "
-	}
-	// check if already has role annotation (/completion makes them)
-	// in that case remove it, and then add to icon
-	// since icon and content are separated by \n
-	contentStr, _ = strings.CutPrefix(contentStr, m.Role+":")
-	// if !strings.HasPrefix(contentStr, m.Role+":") {
-	icon := fmt.Sprintf("(%d) <%s>: ", i, m.Role)
-	// }
-	// Build final message with image indicators before text
-	var finalContent strings.Builder
-	if len(imageIndicators) > 0 {
-		// Add each image indicator on its own line
-		for _, indicator := range imageIndicators {
-			finalContent.WriteString(indicator)
-			finalContent.WriteString("\n")
-		}
-	}
-	finalContent.WriteString(contentStr)
-	textMsg := fmt.Sprintf("[-:-:b]%s[-:-:-]\n%s\n", icon, finalContent.String())
-	return strings.ReplaceAll(textMsg, "\n\n", "\n")
-}
-
 func (m *RoleMsg) ToPrompt() string {
 	var contentStr string
-	if !m.hasContentParts {
+	if !m.HasContentParts {
 		contentStr = m.Content
 	} else {
 		// For structured content, just take the text parts
@@ -289,7 +239,7 @@ func NewRoleMsg(role, content string) RoleMsg {
 	return RoleMsg{
 		Role:            role,
 		Content:         content,
-		hasContentParts: false,
+		HasContentParts: false,
 	}
 }
 
@@ -298,7 +248,7 @@ func NewMultimodalMsg(role string, contentParts []any) RoleMsg {
 	return RoleMsg{
 		Role:            role,
 		ContentParts:    contentParts,
-		hasContentParts: true,
+		HasContentParts: true,
 	}
 }
 
@@ -307,7 +257,7 @@ func (m *RoleMsg) HasContent() bool {
 	if m.Content != "" {
 		return true
 	}
-	if m.hasContentParts && len(m.ContentParts) > 0 {
+	if m.HasContentParts && len(m.ContentParts) > 0 {
 		return true
 	}
 	return false
@@ -315,7 +265,7 @@ func (m *RoleMsg) HasContent() bool {
 
 // IsContentParts returns true if the message uses structured content parts
 func (m *RoleMsg) IsContentParts() bool {
-	return m.hasContentParts
+	return m.HasContentParts
 }
 
 // GetContentParts returns the content parts of the message
@@ -331,38 +281,99 @@ func (m *RoleMsg) Copy() RoleMsg {
 		ContentParts:    m.ContentParts,
 		ToolCallID:      m.ToolCallID,
 		KnownTo:         m.KnownTo,
-		hasContentParts: m.hasContentParts,
+		Stats:           m.Stats,
+		HasContentParts: m.HasContentParts,
+		ToolCall:        m.ToolCall,
+		IsShellCommand:  m.IsShellCommand,
 	}
 }
 
+// GetText returns the text content of the message, handling both
+// simple Content and multimodal ContentParts formats.
+func (m *RoleMsg) GetText() string {
+	if !m.HasContentParts {
+		return m.Content
+	}
+	var textParts []string
+	for _, part := range m.ContentParts {
+		switch p := part.(type) {
+		case TextContentPart:
+			if p.Type == "text" {
+				textParts = append(textParts, p.Text)
+			}
+		case map[string]any:
+			if partType, exists := p["type"]; exists {
+				if partType == "text" {
+					if textVal, textExists := p["text"]; textExists {
+						if textStr, isStr := textVal.(string); isStr {
+							textParts = append(textParts, textStr)
+						}
+					}
+				}
+			}
+		}
+	}
+	return strings.Join(textParts, " ")
+}
+
+// SetText updates the text content of the message. If the message has
+// ContentParts (multimodal), it updates the text parts while preserving
+// images. If not, it sets the simple Content field.
+func (m *RoleMsg) SetText(text string) {
+	if !m.HasContentParts {
+		m.Content = text
+		return
+	}
+	var newParts []any
+	for _, part := range m.ContentParts {
+		switch p := part.(type) {
+		case TextContentPart:
+			if p.Type == "text" {
+				p.Text = text
+				newParts = append(newParts, p)
+			} else {
+				newParts = append(newParts, p)
+			}
+		case map[string]any:
+			if partType, exists := p["type"]; exists && partType == "text" {
+				p["text"] = text
+				newParts = append(newParts, p)
+			} else {
+				newParts = append(newParts, p)
+			}
+		default:
+			newParts = append(newParts, part)
+		}
+	}
+	m.ContentParts = newParts
+}
+
 // AddTextPart adds a text content part to the message
 func (m *RoleMsg) AddTextPart(text string) {
-	if !m.hasContentParts {
+	if !m.HasContentParts {
 		// Convert to content parts format
 		if m.Content != "" {
 			m.ContentParts = []any{TextContentPart{Type: "text", Text: m.Content}}
 		} else {
 			m.ContentParts = []any{}
 		}
-		m.hasContentParts = true
+		m.HasContentParts = true
 	}
-
 	textPart := TextContentPart{Type: "text", Text: text}
 	m.ContentParts = append(m.ContentParts, textPart)
 }
 
 // AddImagePart adds an image content part to the message
 func (m *RoleMsg) AddImagePart(imageURL, imagePath string) {
-	if !m.hasContentParts {
+	if !m.HasContentParts {
 		// Convert to content parts format
 		if m.Content != "" {
 			m.ContentParts = []any{TextContentPart{Type: "text", Text: m.Content}}
 		} else {
 			m.ContentParts = []any{}
 		}
-		m.hasContentParts = true
+		m.HasContentParts = true
 	}
-
 	imagePart := ImageContentPart{
 		Type: "image_url",
 		Path: imagePath, // Store the original file path
@@ -405,31 +416,6 @@ func CreateImageURLFromPath(imagePath string) (string, error) {
 	return fmt.Sprintf("data:%s;base64,%s", mimeType, encoded), nil
 }
 
-// extractDisplayPath returns a path suitable for display, potentially relative to imageBaseDir
-func extractDisplayPath(p string) string {
-	if p == "" {
-		return ""
-	}
-
-	// If base directory is set, try to make path relative to it
-	if imageBaseDir != "" {
-		if rel, err := filepath.Rel(imageBaseDir, p); err == nil {
-			// Check if relative path doesn't start with ".." (meaning it's within base dir)
-			// If it starts with "..", we might still want to show it as relative
-			// but for now we show full path if it goes outside base dir
-			if !strings.HasPrefix(rel, "..") {
-				p = rel
-			}
-		}
-	}
-
-	// Truncate long paths to last 60 characters if needed
-	if len(p) > 60 {
-		return "..." + p[len(p)-60:]
-	}
-	return p
-}
-
 type ChatBody struct {
 	Model    string    `json:"model"`
 	Stream   bool      `json:"stream"`
@@ -437,16 +423,16 @@ type ChatBody struct {
 }
 
 func (cb *ChatBody) Rename(oldname, newname string) {
-	for i, m := range cb.Messages {
+	for i := range cb.Messages {
-		cb.Messages[i].Content = strings.ReplaceAll(m.Content, oldname, newname)
+		cb.Messages[i].Content = strings.ReplaceAll(cb.Messages[i].Content, oldname, newname)
-		cb.Messages[i].Role = strings.ReplaceAll(m.Role, oldname, newname)
+		cb.Messages[i].Role = strings.ReplaceAll(cb.Messages[i].Role, oldname, newname)
 	}
 }
 
 func (cb *ChatBody) ListRoles() []string {
 	namesMap := make(map[string]struct{})
-	for _, m := range cb.Messages {
+	for i := range cb.Messages {
-		namesMap[m.Role] = struct{}{}
+		namesMap[cb.Messages[i].Role] = struct{}{}
 	}
 	resp := make([]string, len(namesMap))
 	i := 0
@@ -643,6 +629,12 @@ func (lcp *LCPModels) ListModels() []string {
 	return resp
 }
 
+type ResponseStats struct {
+	Tokens       int
+	Duration     float64
+	TokensPerSec float64
+}
+
 type ChatRoundReq struct {
 	UserMsg string
 	Role    string

models/models_test.go

@@ -1,167 +0,0 @@
-package models
-
-import (
-	"strings"
-	"testing"
-)
-
-func TestRoleMsgToTextWithImages(t *testing.T) {
-	tests := []struct {
-		name     string
-		msg      RoleMsg
-		index    int
-		expected string // substring to check
-	}{
-		{
-			name:  "text and image",
-			index: 0,
-			msg: func() RoleMsg {
-				msg := NewMultimodalMsg("user", []interface{}{})
-				msg.AddTextPart("Look at this picture")
-				msg.AddImagePart("data:image/jpeg;base64,abc123", "/home/user/Pictures/cat.jpg")
-				return msg
-			}(),
-			expected: "[orange::i][image: /home/user/Pictures/cat.jpg][-:-:-]",
-		},
-		{
-			name:  "image only",
-			index: 1,
-			msg: func() RoleMsg {
-				msg := NewMultimodalMsg("user", []interface{}{})
-				msg.AddImagePart("data:image/png;base64,xyz789", "/tmp/screenshot_20250217_123456.png")
-				return msg
-			}(),
-			expected: "[orange::i][image: /tmp/screenshot_20250217_123456.png][-:-:-]",
-		},
-		{
-			name:  "long filename truncated",
-			index: 2,
-			msg: func() RoleMsg {
-				msg := NewMultimodalMsg("user", []interface{}{})
-				msg.AddTextPart("Check this")
-				msg.AddImagePart("data:image/jpeg;base64,foo", "/very/long/path/to/a/really_long_filename_that_exceeds_forty_characters.jpg")
-				return msg
-			}(),
-			expected: "[orange::i][image: .../to/a/really_long_filename_that_exceeds_forty_characters.jpg][-:-:-]",
-		},
-		{
-			name:  "multiple images",
-			index: 3,
-			msg: func() RoleMsg {
-				msg := NewMultimodalMsg("user", []interface{}{})
-				msg.AddTextPart("Multiple images")
-				msg.AddImagePart("data:image/jpeg;base64,a", "/path/img1.jpg")
-				msg.AddImagePart("data:image/png;base64,b", "/path/img2.png")
-				return msg
-			}(),
-			expected: "[orange::i][image: /path/img1.jpg][-:-:-]\n[orange::i][image: /path/img2.png][-:-:-]",
-		},
-		{
-			name:  "old format without path",
-			index: 4,
-			msg: RoleMsg{
-				Role:            "user",
-				hasContentParts: true,
-				ContentParts: []interface{}{
-					map[string]interface{}{
-						"type": "image_url",
-						"image_url": map[string]interface{}{
-							"url": "data:image/jpeg;base64,old",
-						},
-					},
-				},
-			},
-			expected: "[orange::i][image: image][-:-:-]",
-		},
-		{
-			name:  "old format with path",
-			index: 5,
-			msg: RoleMsg{
-				Role:            "user",
-				hasContentParts: true,
-				ContentParts: []interface{}{
-					map[string]interface{}{
-						"type": "image_url",
-						"path": "/old/path/photo.jpg",
-						"image_url": map[string]interface{}{
-							"url": "data:image/jpeg;base64,old",
-						},
-					},
-				},
-			},
-			expected: "[orange::i][image: /old/path/photo.jpg][-:-:-]",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			result := tt.msg.ToText(tt.index)
-			if !strings.Contains(result, tt.expected) {
-				t.Errorf("ToText() result does not contain expected indicator\ngot: %s\nwant substring: %s", result, tt.expected)
-			}
-			// Ensure the indicator appears before text content
-			if strings.Contains(tt.expected, "cat.jpg") && strings.Contains(result, "Look at this picture") {
-				indicatorPos := strings.Index(result, "[orange::i][image: /home/user/Pictures/cat.jpg][-:-:-]")
-				textPos := strings.Index(result, "Look at this picture")
-				if indicatorPos == -1 || textPos == -1 || indicatorPos >= textPos {
-					t.Errorf("image indicator should appear before text")
-				}
-			}
-		})
-	}
-}
-
-func TestExtractDisplayPath(t *testing.T) {
-	// Save original base dir
-	originalBaseDir := imageBaseDir
-	defer func() { imageBaseDir = originalBaseDir }()
-
-	tests := []struct {
-		name     string
-		baseDir  string
-		path     string
-		expected string
-	}{
-		{
-			name:     "no base dir shows full path",
-			baseDir:  "",
-			path:     "/home/user/images/cat.jpg",
-			expected: "/home/user/images/cat.jpg",
-		},
-		{
-			name:     "relative path within base dir",
-			baseDir:  "/home/user",
-			path:     "/home/user/images/cat.jpg",
-			expected: "images/cat.jpg",
-		},
-		{
-			name:     "path outside base dir shows full path",
-			baseDir:  "/home/user",
-			path:     "/tmp/test.jpg",
-			expected: "/tmp/test.jpg",
-		},
-		{
-			name:     "same directory",
-			baseDir:  "/home/user/images",
-			path:     "/home/user/images/cat.jpg",
-			expected: "cat.jpg",
-		},
-		{
-			name:     "long path truncated",
-			baseDir:  "",
-			path:     "/very/long/path/to/a/really_long_filename_that_exceeds_sixty_characters_limit_yes_it_is_very_long.jpg",
-			expected: "..._that_exceeds_sixty_characters_limit_yes_it_is_very_long.jpg",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			imageBaseDir = tt.baseDir
-			result := extractDisplayPath(tt.path)
-			if result != tt.expected {
-				t.Errorf("extractDisplayPath(%q) with baseDir=%q = %q, want %q",
-					tt.path, tt.baseDir, result, tt.expected)
-			}
-		})
-	}
-}

models/openrouter_test.go

@@ -62,7 +62,6 @@ func TestORModelsListModels(t *testing.T) {
 			t.Errorf("expected 4 total models, got %d", len(allModels))
 		}
 	})
-
 	t.Run("integration with or_models.json", func(t *testing.T) {
 		// Attempt to load the real data file from the project root
 		path := filepath.Join("..", "or_models.json")

popups.go

@@ -51,7 +51,7 @@ func showModelSelectionPopup() {
 	// Find the current model index to set as selected
 	currentModelIndex := -1
 	for i, model := range modelList {
-		if model == chatBody.Model {
+		if strings.TrimPrefix(model, "(loaded) ") == chatBody.Model {
 			currentModelIndex = i
 		}
 		modelListWidget.AddItem(model, "", 0, nil)
@@ -65,16 +65,19 @@ func showModelSelectionPopup() {
 		chatBody.Model = modelName
 		cfg.CurrentModel = chatBody.Model
 		pages.RemovePage("modelSelectionPopup")
+		app.SetFocus(textArea)
 		updateCachedModelColor()
 		updateStatusLine()
 	})
 	modelListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("modelSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("modelSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -160,6 +163,7 @@ func showAPILinkSelectionPopup() {
 			cfg.CurrentModel = chatBody.Model
 		}
 		pages.RemovePage("apiLinkSelectionPopup")
+		app.SetFocus(textArea)
 		choseChunkParser()
 		updateCachedModelColor()
 		updateStatusLine()
@@ -167,10 +171,12 @@ func showAPILinkSelectionPopup() {
 	apiListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("apiLinkSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("apiLinkSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -230,6 +236,7 @@ func showUserRoleSelectionPopup() {
 		textView.SetText(chatToText(filtered, cfg.ShowSys))
 		// Remove the popup page
 		pages.RemovePage("userRoleSelectionPopup")
+		app.SetFocus(textArea)
 		// Update the status line to reflect the change
 		updateStatusLine()
 		colorText()
@@ -237,10 +244,12 @@ func showUserRoleSelectionPopup() {
 	roleListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("userRoleSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("userRoleSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -303,16 +312,19 @@ func showBotRoleSelectionPopup() {
 		cfg.WriteNextMsgAsCompletionAgent = mainText
 		// Remove the popup page
 		pages.RemovePage("botRoleSelectionPopup")
+		app.SetFocus(textArea)
 		// Update the status line to reflect the change
 		updateStatusLine()
 	})
 	roleListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("botRoleSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("botRoleSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -331,7 +343,7 @@ func showBotRoleSelectionPopup() {
 	app.SetFocus(roleListWidget)
 }
 
-func showFileCompletionPopup(filter string) {
+func showShellFileCompletionPopup(filter string) {
 	baseDir := cfg.FilePickerDir
 	if baseDir == "" {
 		baseDir = "."
@@ -340,13 +352,12 @@ func showFileCompletionPopup(filter string) {
 	if len(complMatches) == 0 {
 		return
 	}
-	// If only one match, auto-complete without showing popup
 	if len(complMatches) == 1 {
-		currentText := textArea.GetText()
+		currentText := shellInput.GetText()
 		atIdx := strings.LastIndex(currentText, "@")
 		if atIdx >= 0 {
 			before := currentText[:atIdx]
-			textArea.SetText(before+complMatches[0], true)
+			shellInput.SetText(before + complMatches[0])
 		}
 		return
 	}
@@ -357,21 +368,24 @@ func showFileCompletionPopup(filter string) {
 		widget.AddItem(m, "", 0, nil)
 	}
 	widget.SetSelectedFunc(func(index int, mainText string, secondaryText string, shortcut rune) {
-		currentText := textArea.GetText()
+		currentText := shellInput.GetText()
 		atIdx := strings.LastIndex(currentText, "@")
 		if atIdx >= 0 {
 			before := currentText[:atIdx]
-			textArea.SetText(before+mainText, true)
+			shellInput.SetText(before + mainText)
 		}
-		pages.RemovePage("fileCompletionPopup")
+		pages.RemovePage("shellFileCompletionPopup")
+		app.SetFocus(shellInput)
 	})
 	widget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
-			pages.RemovePage("fileCompletionPopup")
+			pages.RemovePage("shellFileCompletionPopup")
+			app.SetFocus(shellInput)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
-			pages.RemovePage("fileCompletionPopup")
+			pages.RemovePage("shellFileCompletionPopup")
+			app.SetFocus(shellInput)
 			return nil
 		}
 		return event
@@ -385,8 +399,7 @@ func showFileCompletionPopup(filter string) {
 				AddItem(nil, 0, 1, false), width, 1, true).
 			AddItem(nil, 0, 1, false)
 	}
-	// Add modal page and make it visible
+	pages.AddPage("shellFileCompletionPopup", modal(widget, 80, 20), true, true)
-	pages.AddPage("fileCompletionPopup", modal(widget, 80, 20), true, true)
 	app.SetFocus(widget)
 }
 
@@ -395,38 +408,30 @@ func updateWidgetColors(theme *tview.Theme) {
 	fgColor := theme.PrimaryTextColor
 	borderColor := theme.BorderColor
 	titleColor := theme.TitleColor
-
 	textView.SetBackgroundColor(bgColor)
 	textView.SetTextColor(fgColor)
 	textView.SetBorderColor(borderColor)
 	textView.SetTitleColor(titleColor)
-
 	textArea.SetBackgroundColor(bgColor)
 	textArea.SetBorderColor(borderColor)
 	textArea.SetTitleColor(titleColor)
 	textArea.SetTextStyle(tcell.StyleDefault.Background(bgColor).Foreground(fgColor))
 	textArea.SetPlaceholderStyle(tcell.StyleDefault.Background(bgColor).Foreground(fgColor))
-	// Force textarea refresh by restoring text (SetTextStyle doesn't trigger redraw)
 	textArea.SetText(textArea.GetText(), true)
-
 	editArea.SetBackgroundColor(bgColor)
 	editArea.SetBorderColor(borderColor)
 	editArea.SetTitleColor(titleColor)
 	editArea.SetTextStyle(tcell.StyleDefault.Background(bgColor).Foreground(fgColor))
 	editArea.SetPlaceholderStyle(tcell.StyleDefault.Background(bgColor).Foreground(fgColor))
-	// Force textarea refresh by restoring text (SetTextStyle doesn't trigger redraw)
 	editArea.SetText(editArea.GetText(), true)
-
 	statusLineWidget.SetBackgroundColor(bgColor)
 	statusLineWidget.SetTextColor(fgColor)
 	statusLineWidget.SetBorderColor(borderColor)
 	statusLineWidget.SetTitleColor(titleColor)
-
 	helpView.SetBackgroundColor(bgColor)
 	helpView.SetTextColor(fgColor)
 	helpView.SetBorderColor(borderColor)
 	helpView.SetTitleColor(titleColor)
-
 	searchField.SetBackgroundColor(bgColor)
 	searchField.SetBorderColor(borderColor)
 	searchField.SetTitleColor(titleColor)
@@ -453,7 +458,6 @@ func showColorschemeSelectionPopup() {
 	schemeListWidget := tview.NewList().ShowSecondaryText(false).
 		SetSelectedBackgroundColor(tcell.ColorGray)
 	schemeListWidget.SetTitle("Select Colorscheme").SetBorder(true)
-
 	currentScheme := "default"
 	for name := range colorschemes {
 		if tview.Styles == colorschemes[name] {
@@ -484,14 +488,17 @@ func showColorschemeSelectionPopup() {
 		}
 		// Remove the popup page
 		pages.RemovePage("colorschemeSelectionPopup")
+		app.SetFocus(textArea)
 	})
 	schemeListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("colorschemeSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("colorschemeSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event

props_table.go

@@ -115,9 +115,6 @@ func makePropsTable(props map[string]float32) *tview.Table {
 		row++
 	}
 	// Add checkboxes
-	addCheckboxRow("Insert <think> tag (/completion only)", cfg.ThinkUse, func(checked bool) {
-		cfg.ThinkUse = checked
-	})
 	addCheckboxRow("RAG use", cfg.RAGEnabled, func(checked bool) {
 		cfg.RAGEnabled = checked
 	})

rag/embedder.go

@@ -131,7 +131,6 @@ func (a *APIEmbedder) EmbedSlice(lines []string) ([][]float32, error) {
 		}
 		embeddings[data.Index] = data.Embedding
 	}
-
 	return embeddings, nil
 }
 

rag/extractors.go

@@ -0,0 +1,181 @@
+package rag
+
+import (
+	"archive/zip"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/ledongthuc/pdf"
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/parser"
+	"github.com/yuin/goldmark/renderer/html"
+)
+
+func ExtractText(fpath string) (string, error) {
+	ext := strings.ToLower(path.Ext(fpath))
+	switch ext {
+	case ".txt":
+		return extractTextFromFile(fpath)
+	case ".md", ".markdown":
+		return extractTextFromMarkdown(fpath)
+	case ".html", ".htm":
+		return extractTextFromHtmlFile(fpath)
+	case ".epub":
+		return extractTextFromEpub(fpath)
+	case ".pdf":
+		return extractTextFromPdf(fpath)
+	default:
+		return "", fmt.Errorf("unsupported file format: %s", ext)
+	}
+}
+
+func extractTextFromFile(fpath string) (string, error) {
+	data, err := os.ReadFile(fpath)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
+func extractTextFromHtmlFile(fpath string) (string, error) {
+	data, err := os.ReadFile(fpath)
+	if err != nil {
+		return "", err
+	}
+	return extractTextFromHtmlContent(data)
+}
+
+// non utf-8 encoding?
+func extractTextFromHtmlContent(data []byte) (string, error) {
+	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(data))
+	if err != nil {
+		return "", err
+	}
+	// Remove script and style tags
+	doc.Find("script, style, noscript").Each(func(i int, s *goquery.Selection) {
+		s.Remove()
+	})
+	// Get text and clean it
+	text := doc.Text()
+	// Collapse all whitespace (newlines, tabs, multiple spaces) into single spaces
+	cleaned := strings.Join(strings.Fields(text), " ")
+	return cleaned, nil
+}
+
+func extractTextFromMarkdown(fpath string) (string, error) {
+	data, err := os.ReadFile(fpath)
+	if err != nil {
+		return "", err
+	}
+	// Convert markdown to HTML
+	md := goldmark.New(
+		goldmark.WithExtensions(extension.GFM),
+		goldmark.WithParserOptions(parser.WithAutoHeadingID()),
+		goldmark.WithRendererOptions(html.WithUnsafe()), // allow raw HTML if needed
+	)
+	var buf bytes.Buffer
+	if err := md.Convert(data, &buf); err != nil {
+		return "", err
+	}
+	// Now extract text from the resulting HTML (using goquery or similar)
+	return extractTextFromHtmlContent(buf.Bytes())
+}
+
+func extractTextFromEpub(fpath string) (string, error) {
+	r, err := zip.OpenReader(fpath)
+	if err != nil {
+		return "", fmt.Errorf("failed to open epub: %w", err)
+	}
+	defer r.Close()
+	var sb strings.Builder
+	for _, f := range r.File {
+		ext := strings.ToLower(path.Ext(f.Name))
+		if ext != ".xhtml" && ext != ".html" && ext != ".htm" && ext != ".xml" {
+			continue
+		}
+
+		// Skip manifest, toc, ncx files - they don't contain book content
+		nameLower := strings.ToLower(f.Name)
+		if strings.Contains(nameLower, "toc") || strings.Contains(nameLower, "nav") ||
+			strings.Contains(nameLower, "manifest") || strings.Contains(nameLower, ".opf") ||
+			strings.HasSuffix(nameLower, ".ncx") {
+			continue
+		}
+
+		rc, err := f.Open()
+		if err != nil {
+			continue
+		}
+
+		if sb.Len() > 0 {
+			sb.WriteString("\n\n")
+		}
+		sb.WriteString(f.Name)
+		sb.WriteString("\n")
+
+		buf, readErr := io.ReadAll(rc)
+		rc.Close()
+		if readErr == nil {
+			sb.WriteString(stripHTML(string(buf)))
+		}
+	}
+	if sb.Len() == 0 {
+		return "", errors.New("no content extracted from epub")
+	}
+	return sb.String(), nil
+}
+
+func stripHTML(html string) string {
+	var sb strings.Builder
+	inTag := false
+	for _, r := range html {
+		switch r {
+		case '<':
+			inTag = true
+		case '>':
+			inTag = false
+		default:
+			if !inTag {
+				sb.WriteRune(r)
+			}
+		}
+	}
+	return sb.String()
+}
+
+func extractTextFromPdf(fpath string) (string, error) {
+	_, err := exec.LookPath("pdftotext")
+	if err == nil {
+		out, err := exec.Command("pdftotext", "-layout", fpath, "-").Output()
+		if err == nil && len(out) > 0 {
+			return string(out), nil
+		}
+	}
+	return extractTextFromPdfPureGo(fpath)
+}
+
+func extractTextFromPdfPureGo(fpath string) (string, error) {
+	df, r, err := pdf.Open(fpath)
+	if err != nil {
+		return "", fmt.Errorf("failed to open pdf: %w", err)
+	}
+	defer df.Close()
+	textReader, err := r.GetPlainText()
+	if err != nil {
+		return "", fmt.Errorf("failed to extract text from pdf: %w", err)
+	}
+	var buf bytes.Buffer
+	_, err = io.Copy(&buf, textReader)
+	if err != nil {
+		return "", fmt.Errorf("failed to read pdf text: %w", err)
+	}
+	return buf.String(), nil
+}

rag/rag.go

@@ -7,8 +7,9 @@ import (
 	"gf-lt/models"
 	"gf-lt/storage"
 	"log/slog"
-	"os"
 	"path"
+	"regexp"
+	"sort"
 	"strings"
 	"sync"
 
@@ -23,19 +24,18 @@ var (
 	ErrRAGStatus        = "some error occurred; failed to transfer data to vector db"
 )
 
-
 type RAG struct {
 	logger   *slog.Logger
 	store    storage.FullRepo
 	cfg      *config.Config
 	embedder Embedder
 	storage  *VectorStorage
+	mu       sync.Mutex
 }
 
 func New(l *slog.Logger, s storage.FullRepo, cfg *config.Config) *RAG {
 	// Initialize with API embedder by default, could be configurable later
 	embedder := NewAPIEmbedder(l, cfg)
-
 	rag := &RAG{
 		logger:   l,
 		store:    s,
@@ -54,7 +54,9 @@ func wordCounter(sentence string) int {
 }
 
 func (r *RAG) LoadRAG(fpath string) error {
-	data, err := os.ReadFile(fpath)
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	fileText, err := ExtractText(fpath)
 	if err != nil {
 		return err
 	}
@@ -63,10 +65,7 @@ func (r *RAG) LoadRAG(fpath string) error {
 	case LongJobStatusCh <- LoadedFileRAGStatus:
 	default:
 		r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", LoadedFileRAGStatus)
-		// Channel is full or closed, ignore the message to prevent panic
 	}
-
-	fileText := string(data)
 	tokenizer, err := english.NewSentenceTokenizer(nil)
 	if err != nil {
 		return err
@@ -76,19 +75,16 @@ func (r *RAG) LoadRAG(fpath string) error {
 	for i, s := range sentences {
 		sents[i] = s.Text
 	}
-
 	// Group sentences into paragraphs based on word limit
 	paragraphs := []string{}
 	par := strings.Builder{}
 	for i := 0; i < len(sents); i++ {
-		// Only add sentences that aren't empty
 		if strings.TrimSpace(sents[i]) != "" {
 			if par.Len() > 0 {
-				par.WriteString(" ") // Add space between sentences
+				par.WriteString(" ")
 			}
 			par.WriteString(sents[i])
 		}
-
 		if wordCounter(par.String()) > int(r.cfg.RAGWordLimit) {
 			paragraph := strings.TrimSpace(par.String())
 			if paragraph != "" {
@@ -97,7 +93,6 @@ func (r *RAG) LoadRAG(fpath string) error {
 			par.Reset()
 		}
 	}
-
 	// Handle any remaining content in the paragraph buffer
 	if par.Len() > 0 {
 		paragraph := strings.TrimSpace(par.String())
@@ -105,215 +100,82 @@ func (r *RAG) LoadRAG(fpath string) error {
 			paragraphs = append(paragraphs, paragraph)
 		}
 	}
-
 	// Adjust batch size if needed
 	if len(paragraphs) < r.cfg.RAGBatchSize && len(paragraphs) > 0 {
 		r.cfg.RAGBatchSize = len(paragraphs)
 	}
-
 	if len(paragraphs) == 0 {
 		return errors.New("no valid paragraphs found in file")
 	}
-
+	// Process paragraphs in batches synchronously
-	var (
+	batchCount := 0
-		maxChSize = 100
+	for i := 0; i < len(paragraphs); i += r.cfg.RAGBatchSize {
-		left      = 0
+		end := i + r.cfg.RAGBatchSize
-		right     = r.cfg.RAGBatchSize
+		if end > len(paragraphs) {
-		batchCh   = make(chan map[int][]string, maxChSize)
+			end = len(paragraphs)
-		vectorCh  = make(chan []models.VectorRow, maxChSize)
-		errCh     = make(chan error, 1)
-		wg        = new(sync.WaitGroup)
-		lock      = new(sync.Mutex)
-	)
-
-	defer close(errCh)
-	defer close(batchCh)
-
-	// Fill input channel with batches
-	ctn := 0
-	totalParagraphs := len(paragraphs)
-	for {
-		if right > totalParagraphs {
-			batchCh <- map[int][]string{left: paragraphs[left:]}
-			break
 		}
-		batchCh <- map[int][]string{left: paragraphs[left:right]}
+		batch := paragraphs[i:end]
-		left, right = right, right+r.cfg.RAGBatchSize
+		batchCount++
-		ctn++
+		// Filter empty paragraphs
-	}
+		nonEmptyBatch := make([]string, 0, len(batch))
-
+		for _, p := range batch {
-	finishedBatchesMsg := fmt.Sprintf("finished batching batches#: %d; paragraphs: %d; sentences: %d\n", ctn+1, len(paragraphs), len(sents))
+			if strings.TrimSpace(p) != "" {
-	r.logger.Debug(finishedBatchesMsg)
+				nonEmptyBatch = append(nonEmptyBatch, strings.TrimSpace(p))
-	select {
+			}
-	case LongJobStatusCh <- finishedBatchesMsg:
+		}
-	default:
+		if len(nonEmptyBatch) == 0 {
-		r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", finishedBatchesMsg)
+			continue
-		// Channel is full or closed, ignore the message to prevent panic
+		}
-	}
+		// Embed the batch
-
+		embeddings, err := r.embedder.EmbedSlice(nonEmptyBatch)
-	// Start worker goroutines with WaitGroup
-	wg.Add(int(r.cfg.RAGWorkers))
-	for w := 0; w < int(r.cfg.RAGWorkers); w++ {
-		go func(workerID int) {
-			defer wg.Done()
-			r.batchToVectorAsync(lock, workerID, batchCh, vectorCh, errCh, path.Base(fpath))
-		}(w)
-	}
-
-	// Use a goroutine to close the batchCh when all batches are sent
-	go func() {
-		wg.Wait()
-		close(vectorCh) // Close vectorCh when all workers are done
-	}()
-
-	// Check for errors from workers
-	// Use a non-blocking check for errors
-	select {
-	case err := <-errCh:
 		if err != nil {
-			r.logger.Error("error during RAG processing", "error", err)
+			r.logger.Error("failed to embed batch", "error", err, "batch", batchCount)
+			select {
+			case LongJobStatusCh <- ErrRAGStatus:
+			default:
+				r.logger.Warn("LongJobStatusCh channel full, dropping message")
+			}
+			return fmt.Errorf("failed to embed batch %d: %w", batchCount, err)
+		}
+		if len(embeddings) != len(nonEmptyBatch) {
+			err := errors.New("embedding count mismatch")
+			r.logger.Error("embedding mismatch", "expected", len(nonEmptyBatch), "got", len(embeddings))
 			return err
 		}
-	default:
+		// Write vectors to storage
-		// No immediate error, continue
+		filename := path.Base(fpath)
-	}
+		for j, text := range nonEmptyBatch {
-
+			vector := models.VectorRow{
-	// Write vectors to storage - this will block until vectorCh is closed
+				Embeddings: embeddings[j],
-	return r.writeVectors(vectorCh)
+				RawText:    text,
-}
+				Slug:       fmt.Sprintf("%s_%d_%d", filename, batchCount, j),
-
+				FileName:   filename,
-func (r *RAG) writeVectors(vectorCh chan []models.VectorRow) error {
-	for {
-		for batch := range vectorCh {
-			for _, vector := range batch {
-				if err := r.storage.WriteVector(&vector); err != nil {
-					r.logger.Error("failed to write vector to DB", "error", err, "slug", vector.Slug)
-					select {
-					case LongJobStatusCh <- ErrRAGStatus:
-					default:
-						r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", ErrRAGStatus)
-						// Channel is full or closed, ignore the message to prevent panic
-					}
-					return err // Stop the entire RAG operation on DB error
-				}
 			}
-			r.logger.Debug("wrote batch to db", "size", len(batch), "vector_chan_len", len(vectorCh))
+			if err := r.storage.WriteVector(&vector); err != nil {
-			if len(vectorCh) == 0 {
+				r.logger.Error("failed to write vector to DB", "error", err, "slug", vector.Slug)
-				r.logger.Debug("finished writing vectors")
 				select {
-				case LongJobStatusCh <- FinishedRAGStatus:
+				case LongJobStatusCh <- ErrRAGStatus:
 				default:
-					r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", FinishedRAGStatus)
+					r.logger.Warn("LongJobStatusCh channel full, dropping message")
-					// Channel is full or closed, ignore the message to prevent panic
 				}
-				return nil
+				return fmt.Errorf("failed to write vector: %w", err)
 			}
 		}
-	}
+		r.logger.Debug("wrote batch to db", "batch", batchCount, "size", len(nonEmptyBatch))
-}
+		// Send progress status
-
+		statusMsg := fmt.Sprintf("processed batch %d/%d", batchCount, (len(paragraphs)+r.cfg.RAGBatchSize-1)/r.cfg.RAGBatchSize)
-func (r *RAG) batchToVectorAsync(lock *sync.Mutex, id int, inputCh <-chan map[int][]string,
-	vectorCh chan<- []models.VectorRow, errCh chan error, filename string) {
-	var err error
-
-	defer func() {
-		// For errCh, make sure we only send if there's actually an error and the channel can accept it
-		if err != nil {
-			select {
-			case errCh <- err:
-			default:
-				// errCh might be full or closed, log but don't panic
-				r.logger.Warn("errCh channel full or closed, skipping error propagation", "worker", id, "error", err)
-			}
-		}
-	}()
-
-	for {
-		lock.Lock()
-		if len(inputCh) == 0 {
-			lock.Unlock()
-			return
-		}
-
-		select {
-		case linesMap := <-inputCh:
-			for leftI, lines := range linesMap {
-				if err := r.fetchEmb(lines, errCh, vectorCh, fmt.Sprintf("%s_%d", filename, leftI), filename); err != nil {
-					r.logger.Error("error fetching embeddings", "error", err, "worker", id)
-					lock.Unlock()
-					return
-				}
-			}
-			lock.Unlock()
-		case err = <-errCh:
-			r.logger.Error("got an error from error channel", "error", err)
-			lock.Unlock()
-			return
-		default:
-			lock.Unlock()
-		}
-
-		r.logger.Debug("processed batch", "batches#", len(inputCh), "worker#", id)
-		statusMsg := fmt.Sprintf("converted to vector; batches: %d, worker#: %d", len(inputCh), id)
 		select {
 		case LongJobStatusCh <- statusMsg:
 		default:
-			r.logger.Warn("LongJobStatusCh channel full or closed, dropping status message", "message", statusMsg)
+			r.logger.Warn("LongJobStatusCh channel full, dropping message")
-			// Channel is full or closed, ignore the message to prevent panic
 		}
 	}
-}
+	r.logger.Debug("finished writing vectors", "batches", batchCount)
-
+	select {
-func (r *RAG) fetchEmb(lines []string, errCh chan error, vectorCh chan<- []models.VectorRow, slug, filename string) error {
+	case LongJobStatusCh <- FinishedRAGStatus:
-	// Filter out empty lines before sending to embedder
+	default:
-	nonEmptyLines := make([]string, 0, len(lines))
+		r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", FinishedRAGStatus)
-	for _, line := range lines {
-		trimmed := strings.TrimSpace(line)
-		if trimmed != "" {
-			nonEmptyLines = append(nonEmptyLines, trimmed)
-		}
 	}
-
-	// Skip if no non-empty lines
-	if len(nonEmptyLines) == 0 {
-		// Send empty result but don't error
-		vectorCh <- []models.VectorRow{}
-		return nil
-	}
-
-	embeddings, err := r.embedder.EmbedSlice(nonEmptyLines)
-	if err != nil {
-		r.logger.Error("failed to embed lines", "err", err.Error())
-		errCh <- err
-		return err
-	}
-
-	if len(embeddings) == 0 {
-		err := errors.New("no embeddings returned")
-		r.logger.Error("empty embeddings")
-		errCh <- err
-		return err
-	}
-
-	if len(embeddings) != len(nonEmptyLines) {
-		err := errors.New("mismatch between number of lines and embeddings returned")
-		r.logger.Error("embedding mismatch", "err", err.Error())
-		errCh <- err
-		return err
-	}
-
-	// Create a VectorRow for each line in the batch
-	vectors := make([]models.VectorRow, len(nonEmptyLines))
-	for i, line := range nonEmptyLines {
-		vectors[i] = models.VectorRow{
-			Embeddings: embeddings[i],
-			RawText:    line,
-			Slug:       fmt.Sprintf("%s_%d", slug, i),
-			FileName:   filename,
-		}
-	}
-
-	vectorCh <- vectors
 	return nil
 }
 
@@ -332,3 +194,259 @@ func (r *RAG) ListLoaded() ([]string, error) {
 func (r *RAG) RemoveFile(filename string) error {
 	return r.storage.RemoveEmbByFileName(filename)
 }
+
+var (
+	queryRefinementPattern = regexp.MustCompile(`(?i)(based on my (vector db|vector db|vector database|rags?|past (conversations?|chat|messages?))|from my (files?|documents?|data|information|memory)|search (in|my) (vector db|database|rags?)|rag search for)`)
+	importantKeywords      = []string{"project", "architecture", "code", "file", "chat", "conversation", "topic", "summary", "details", "history", "previous", "my", "user", "me"}
+	stopWords              = []string{"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", "from", "up", "down", "left", "right"}
+)
+
+func (r *RAG) RefineQuery(query string) string {
+	original := query
+	query = strings.TrimSpace(query)
+	if len(query) == 0 {
+		return original
+	}
+	if len(query) <= 3 {
+		return original
+	}
+	query = strings.ToLower(query)
+	for _, stopWord := range stopWords {
+		wordPattern := `\b` + stopWord + `\b`
+		re := regexp.MustCompile(wordPattern)
+		query = re.ReplaceAllString(query, "")
+	}
+	query = strings.TrimSpace(query)
+	if len(query) < 5 {
+		return original
+	}
+	if queryRefinementPattern.MatchString(original) {
+		cleaned := queryRefinementPattern.ReplaceAllString(original, "")
+		cleaned = strings.TrimSpace(cleaned)
+		if len(cleaned) >= 5 {
+			return cleaned
+		}
+	}
+	query = r.extractImportantPhrases(query)
+	if len(query) < 5 {
+		return original
+	}
+	return query
+}
+
+func (r *RAG) extractImportantPhrases(query string) string {
+	words := strings.Fields(query)
+	var important []string
+	for _, word := range words {
+		word = strings.Trim(word, ".,!?;:'\"()[]{}")
+		isImportant := false
+		for _, kw := range importantKeywords {
+			if strings.Contains(strings.ToLower(word), kw) {
+				isImportant = true
+				break
+			}
+		}
+		if isImportant || len(word) > 3 {
+			important = append(important, word)
+		}
+	}
+	if len(important) == 0 {
+		return query
+	}
+	return strings.Join(important, " ")
+}
+
+func (r *RAG) GenerateQueryVariations(query string) []string {
+	variations := []string{query}
+	if len(query) < 5 {
+		return variations
+	}
+	parts := strings.Fields(query)
+	if len(parts) == 0 {
+		return variations
+	}
+	if len(parts) >= 2 {
+		trimmed := strings.Join(parts[:len(parts)-1], " ")
+		if len(trimmed) >= 5 {
+			variations = append(variations, trimmed)
+		}
+	}
+	if len(parts) >= 2 {
+		trimmed := strings.Join(parts[1:], " ")
+		if len(trimmed) >= 5 {
+			variations = append(variations, trimmed)
+		}
+	}
+	if !strings.HasSuffix(query, " explanation") {
+		variations = append(variations, query+" explanation")
+	}
+	if !strings.HasPrefix(query, "what is ") {
+		variations = append(variations, "what is "+query)
+	}
+	if !strings.HasSuffix(query, " details") {
+		variations = append(variations, query+" details")
+	}
+	if !strings.HasSuffix(query, " summary") {
+		variations = append(variations, query+" summary")
+	}
+	return variations
+}
+
+func (r *RAG) RerankResults(results []models.VectorRow, query string) []models.VectorRow {
+	type scoredResult struct {
+		row      models.VectorRow
+		distance float32
+	}
+	scored := make([]scoredResult, 0, len(results))
+	for i := range results {
+		row := results[i]
+
+		score := float32(0)
+		rawTextLower := strings.ToLower(row.RawText)
+		queryLower := strings.ToLower(query)
+		if strings.Contains(rawTextLower, queryLower) {
+			score += 10
+		}
+		queryWords := strings.Fields(queryLower)
+		matchCount := 0
+		for _, word := range queryWords {
+			if len(word) > 2 && strings.Contains(rawTextLower, word) {
+				matchCount++
+			}
+		}
+		if len(queryWords) > 0 {
+			score += float32(matchCount) / float32(len(queryWords)) * 5
+		}
+		if row.FileName == "chat" || strings.Contains(strings.ToLower(row.FileName), "conversation") {
+			score += 3
+		}
+		distance := row.Distance - score/100
+		scored = append(scored, scoredResult{row: row, distance: distance})
+	}
+	sort.Slice(scored, func(i, j int) bool {
+		return scored[i].distance < scored[j].distance
+	})
+	unique := make([]models.VectorRow, 0)
+	seen := make(map[string]bool)
+	for i := range scored {
+		if !seen[scored[i].row.Slug] {
+			seen[scored[i].row.Slug] = true
+			unique = append(unique, scored[i].row)
+		}
+	}
+	if len(unique) > 10 {
+		unique = unique[:10]
+	}
+	return unique
+}
+
+func (r *RAG) SynthesizeAnswer(results []models.VectorRow, query string) (string, error) {
+	if len(results) == 0 {
+		return "No relevant information found in the vector database.", nil
+	}
+	var contextBuilder strings.Builder
+	contextBuilder.WriteString("User Query: ")
+	contextBuilder.WriteString(query)
+	contextBuilder.WriteString("\n\nRetrieved Context:\n")
+	for i, row := range results {
+		fmt.Fprintf(&contextBuilder, "[Source %d: %s]\n", i+1, row.FileName)
+		contextBuilder.WriteString(row.RawText)
+		contextBuilder.WriteString("\n\n")
+	}
+	contextBuilder.WriteString("Instructions: ")
+	contextBuilder.WriteString("Based on the retrieved context above, provide a concise, coherent answer to the user's query. ")
+	contextBuilder.WriteString("Extract only the most relevant information. ")
+	contextBuilder.WriteString("If no relevant information is found, state that clearly. ")
+	contextBuilder.WriteString("Cite sources by filename when relevant. ")
+	contextBuilder.WriteString("Do not include unnecessary preamble or explanations.")
+	synthesisPrompt := contextBuilder.String()
+	emb, err := r.LineToVector(synthesisPrompt)
+	if err != nil {
+		r.logger.Error("failed to embed synthesis prompt", "error", err)
+		return "", err
+	}
+	embResp := &models.EmbeddingResp{
+		Embedding: emb,
+		Index:     0,
+	}
+	topResults, err := r.SearchEmb(embResp)
+	if err != nil {
+		r.logger.Error("failed to search for synthesis context", "error", err)
+		return "", err
+	}
+	if len(topResults) > 0 && topResults[0].RawText != synthesisPrompt {
+		return topResults[0].RawText, nil
+	}
+	var finalAnswer strings.Builder
+	finalAnswer.WriteString("Based on the retrieved context:\n\n")
+	for i, row := range results {
+		if i >= 5 {
+			break
+		}
+		fmt.Fprintf(&finalAnswer, "- From %s: %s\n", row.FileName, truncateString(row.RawText, 200))
+	}
+	return finalAnswer.String(), nil
+}
+
+func truncateString(s string, maxLen int) string {
+	if len(s) <= maxLen {
+		return s
+	}
+	return s[:maxLen] + "..."
+}
+
+func (r *RAG) Search(query string, limit int) ([]models.VectorRow, error) {
+	refined := r.RefineQuery(query)
+	variations := r.GenerateQueryVariations(refined)
+	allResults := make([]models.VectorRow, 0)
+	seen := make(map[string]bool)
+	for _, q := range variations {
+		emb, err := r.LineToVector(q)
+		if err != nil {
+			r.logger.Error("failed to embed query variation", "error", err, "query", q)
+			continue
+		}
+
+		embResp := &models.EmbeddingResp{
+			Embedding: emb,
+			Index:     0,
+		}
+
+		results, err := r.SearchEmb(embResp)
+		if err != nil {
+			r.logger.Error("failed to search embeddings", "error", err, "query", q)
+			continue
+		}
+
+		for _, row := range results {
+			if !seen[row.Slug] {
+				seen[row.Slug] = true
+				allResults = append(allResults, row)
+			}
+		}
+	}
+	reranked := r.RerankResults(allResults, query)
+	if len(reranked) > limit {
+		reranked = reranked[:limit]
+	}
+	return reranked, nil
+}
+
+var (
+	ragInstance *RAG
+	ragOnce     sync.Once
+)
+
+func Init(c *config.Config, l *slog.Logger, s storage.FullRepo) error {
+	ragOnce.Do(func() {
+		if c == nil || l == nil || s == nil {
+			return
+		}
+		ragInstance = New(l, s, c)
+	})
+	return nil
+}
+
+func GetInstance() *RAG {
+	return ragInstance
+}

rag/storage.go

@@ -28,7 +28,6 @@ func NewVectorStorage(logger *slog.Logger, store storage.FullRepo) *VectorStorag
 	}
 }
 
-
 // SerializeVector converts []float32 to binary blob
 func SerializeVector(vec []float32) []byte {
 	buf := make([]byte, len(vec)*4) // 4 bytes per float32
@@ -66,17 +65,14 @@ func (vs *VectorStorage) WriteVector(row *models.VectorRow) error {
 
 	// Serialize the embeddings to binary
 	serializedEmbeddings := SerializeVector(row.Embeddings)
-
 	query := fmt.Sprintf(
 		"INSERT INTO %s (embeddings, slug, raw_text, filename) VALUES (?, ?, ?, ?)",
 		tableName,
 	)
-
 	if _, err := vs.sqlxDB.Exec(query, serializedEmbeddings, row.Slug, row.RawText, row.FileName); err != nil {
 		vs.logger.Error("failed to write vector", "error", err, "slug", row.Slug)
 		return err
 	}
-
 	return nil
 }
 
@@ -86,20 +82,18 @@ func (vs *VectorStorage) getTableName(emb []float32) (string, error) {
 
 	// Check if we support this embedding size
 	supportedSizes := map[int]bool{
-		384:   true,
+		384:  true,
-		768:   true,
+		768:  true,
-		1024:  true,
+		1024: true,
-		1536:  true,
+		1536: true,
-		2048:  true,
+		2048: true,
-		3072:  true,
+		3072: true,
-		4096:  true,
+		4096: true,
-		5120:  true,
+		5120: true,
 	}
-
 	if supportedSizes[size] {
 		return fmt.Sprintf("embeddings_%d", size), nil
 	}
-
 	return "", fmt.Errorf("no table for embedding size of %d", size)
 }
 
@@ -126,9 +120,7 @@ func (vs *VectorStorage) SearchClosest(query []float32) ([]models.VectorRow, err
 		vector   models.VectorRow
 		distance float32
 	}
-
 	var topResults []SearchResult
-
 	// Process vectors one by one to avoid loading everything into memory
 	for rows.Next() {
 		var (
@@ -176,14 +168,12 @@ func (vs *VectorStorage) SearchClosest(query []float32) ([]models.VectorRow, err
 		result.vector.Distance = result.distance
 		results = append(results, result.vector)
 	}
-
 	return results, nil
 }
 
 // ListFiles returns a list of all loaded files
 func (vs *VectorStorage) ListFiles() ([]string, error) {
 	fileLists := make([][]string, 0)
-
 	// Query all supported tables and combine results
 	embeddingSizes := []int{384, 768, 1024, 1536, 2048, 3072, 4096, 5120}
 	for _, size := range embeddingSizes {
@@ -219,14 +209,12 @@ func (vs *VectorStorage) ListFiles() ([]string, error) {
 			}
 		}
 	}
-
 	return allFiles, nil
 }
 
 // RemoveEmbByFileName removes all embeddings associated with a specific filename
 func (vs *VectorStorage) RemoveEmbByFileName(filename string) error {
 	var errors []string
-
 	embeddingSizes := []int{384, 768, 1024, 1536, 2048, 3072, 4096, 5120}
 	for _, size := range embeddingSizes {
 		table := fmt.Sprintf("embeddings_%d", size)
@@ -235,11 +223,9 @@ func (vs *VectorStorage) RemoveEmbByFileName(filename string) error {
 			errors = append(errors, err.Error())
 		}
 	}
-
 	if len(errors) > 0 {
 		return fmt.Errorf("errors occurred: %s", strings.Join(errors, "; "))
 	}
-
 	return nil
 }
 
@@ -248,18 +234,15 @@ func cosineSimilarity(a, b []float32) float32 {
 	if len(a) != len(b) {
 		return 0.0
 	}
-
 	var dotProduct, normA, normB float32
 	for i := 0; i < len(a); i++ {
 		dotProduct += a[i] * b[i]
 		normA += a[i] * a[i]
 		normB += b[i] * b[i]
 	}
-
 	if normA == 0 || normB == 0 {
 		return 0.0
 	}
-
 	return dotProduct / (sqrt(normA) * sqrt(normB))
 }
 
@@ -275,4 +258,3 @@ func sqrt(f float32) float32 {
 	}
 	return guess
 }
-

session.go

@@ -131,13 +131,18 @@ func loadOldChatOrGetNew() []models.RoleMsg {
 	chat, err := store.GetLastChat()
 	if err != nil {
 		logger.Warn("failed to load history chat", "error", err)
+		maxID, err := store.ChatGetMaxID()
+		if err != nil {
+			logger.Error("failed to fetch max chat id", "error", err)
+		}
+		maxID++
 		chat := &models.Chat{
-			ID:        0,
+			ID:        maxID,
 			CreatedAt: time.Now(),
 			UpdatedAt: time.Now(),
 			Agent:     cfg.AssistantRole,
 		}
-		chat.Name = fmt.Sprintf("%s_%v", chat.Agent, chat.CreatedAt.Unix())
+		chat.Name = fmt.Sprintf("%s_%v", chat.Agent, chat.ID)
 		activeChatName = chat.Name
 		chatMap[chat.Name] = chat
 		return defaultStarter
@@ -149,10 +154,6 @@ func loadOldChatOrGetNew() []models.RoleMsg {
 		chatMap[chat.Name] = chat
 		return defaultStarter
 	}
-	// if chat.Name == "" {
-	// 	logger.Warn("empty chat name", "id", chat.ID)
-	// 	chat.Name = fmt.Sprintf("%s_%v", chat.Agent, chat.CreatedAt.Unix())
-	// }
 	chatMap[chat.Name] = chat
 	activeChatName = chat.Name
 	cfg.AssistantRole = chat.Agent

storage/storage.go

@@ -103,7 +103,6 @@ func NewProviderSQL(dbPath string, logger *slog.Logger) FullRepo {
 		return nil
 	}
 	p := ProviderSQL{db: db, logger: logger}
-
 	p.Migrate()
 	return p
 }

storage/vector.go

@@ -73,12 +73,9 @@ func (p ProviderSQL) WriteVector(row *models.VectorRow) error {
 	if err != nil {
 		return err
 	}
-
 	serializedEmbeddings := SerializeVector(row.Embeddings)
-
 	query := fmt.Sprintf("INSERT INTO %s(embeddings, slug, raw_text, filename) VALUES (?, ?, ?, ?)", tableName)
 	_, err = p.db.Exec(query, serializedEmbeddings, row.Slug, row.RawText, row.FileName)
-
 	return err
 }
 
@@ -87,27 +84,22 @@ func (p ProviderSQL) SearchClosest(q []float32) ([]models.VectorRow, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	querySQL := "SELECT embeddings, slug, raw_text, filename FROM " + tableName
 	rows, err := p.db.Query(querySQL)
 	if err != nil {
 		return nil, err
 	}
 	defer rows.Close()
-
 	type SearchResult struct {
 		vector   models.VectorRow
 		distance float32
 	}
-
 	var topResults []SearchResult
-
 	for rows.Next() {
 		var (
-			embeddingsBlob []byte
+			embeddingsBlob          []byte
 			slug, rawText, fileName string
 		)
-
 		if err := rows.Scan(&embeddingsBlob, &slug, &rawText, &fileName); err != nil {
 			continue
 		}
@@ -152,7 +144,6 @@ func (p ProviderSQL) SearchClosest(q []float32) ([]models.VectorRow, error) {
 		result.vector.Distance = result.distance
 		results[i] = result.vector
 	}
-
 	return results, nil
 }
 
@@ -161,18 +152,15 @@ func cosineSimilarity(a, b []float32) float32 {
 	if len(a) != len(b) {
 		return 0.0
 	}
-
 	var dotProduct, normA, normB float32
 	for i := 0; i < len(a); i++ {
 		dotProduct += a[i] * b[i]
 		normA += a[i] * a[i]
 		normB += b[i] * b[i]
 	}
-
 	if normA == 0 || normB == 0 {
 		return 0.0
 	}
-
 	return dotProduct / (sqrt(normA) * sqrt(normB))
 }
 
@@ -229,13 +217,11 @@ func (p ProviderSQL) ListFiles() ([]string, error) {
 			}
 		}
 	}
-
 	return allFiles, nil
 }
 
 func (p ProviderSQL) RemoveEmbByFileName(filename string) error {
 	var errors []string
-
 	tableNames := []string{
 		"embeddings_384", "embeddings_768", "embeddings_1024", "embeddings_1536",
 		"embeddings_2048", "embeddings_3072", "embeddings_4096", "embeddings_5120",
@@ -246,10 +232,8 @@ func (p ProviderSQL) RemoveEmbByFileName(filename string) error {
 			errors = append(errors, err.Error())
 		}
 	}
-
 	if len(errors) > 0 {
 		return fmt.Errorf("errors occurred: %v", errors)
 	}
-
 	return nil
 }

tables.go

@@ -236,9 +236,59 @@ func makeChatTable(chatMap map[string]models.Chat) *tview.Table {
 }
 
 // nolint:unused
-func makeRAGTable(fileList []string) *tview.Flex {
+func formatSize(size int64) string {
-	actions := []string{"load", "delete"}
+	units := []string{"B", "KB", "MB", "GB", "TB"}
-	rows, cols := len(fileList), len(actions)+1
+	i := 0
+	s := float64(size)
+	for s >= 1024 && i < len(units)-1 {
+		s /= 1024
+		i++
+	}
+	return fmt.Sprintf("%.1f%s", s, units[i])
+}
+
+type ragFileInfo struct {
+	name     string
+	inRAGDir bool
+	isLoaded bool
+	fullPath string
+}
+
+func makeRAGTable(fileList []string, loadedFiles []string) *tview.Flex {
+	// Build set of loaded files for quick lookup
+	loadedSet := make(map[string]bool)
+	for _, f := range loadedFiles {
+		loadedSet[f] = true
+	}
+
+	// Build merged list: files from ragdir + orphaned files from DB
+	ragFiles := make([]ragFileInfo, 0, len(fileList)+len(loadedFiles))
+	seen := make(map[string]bool)
+
+	// Add files from ragdir
+	for _, f := range fileList {
+		ragFiles = append(ragFiles, ragFileInfo{
+			name:     f,
+			inRAGDir: true,
+			isLoaded: loadedSet[f],
+			fullPath: path.Join(cfg.RAGDir, f),
+		})
+		seen[f] = true
+	}
+
+	// Add orphaned files (in DB but not in ragdir)
+	for _, f := range loadedFiles {
+		if !seen[f] {
+			ragFiles = append(ragFiles, ragFileInfo{
+				name:     f,
+				inRAGDir: false,
+				isLoaded: true,
+				fullPath: "",
+			})
+		}
+	}
+	rows := len(ragFiles)
+	cols := 4 // File Name | Preview | Action | Delete
 	fileTable := tview.NewTable().
 		SetBorders(true)
 	longStatusView := tview.NewTextView()
@@ -252,41 +302,92 @@ func makeRAGTable(fileList []string) *tview.Flex {
 		AddItem(fileTable, 0, 60, true)
 	// Add the exit option as the first row (row 0)
 	fileTable.SetCell(0, 0,
-		tview.NewTableCell("Exit RAG manager").
+		tview.NewTableCell("File Name").
 			SetTextColor(tcell.ColorWhite).
 			SetAlign(tview.AlignCenter).
 			SetSelectable(false))
 	fileTable.SetCell(0, 1,
-		tview.NewTableCell("(Close without action)").
+		tview.NewTableCell("Preview").
-			SetTextColor(tcell.ColorGray).
+			SetTextColor(tcell.ColorWhite).
 			SetAlign(tview.AlignCenter).
 			SetSelectable(false))
 	fileTable.SetCell(0, 2,
-		tview.NewTableCell("exit").
+		tview.NewTableCell("Load/Unload").
-			SetTextColor(tcell.ColorGray).
+			SetTextColor(tcell.ColorWhite).
-			SetAlign(tview.AlignCenter))
+			SetAlign(tview.AlignCenter).
+			SetSelectable(false))
+	fileTable.SetCell(0, 3,
+		tview.NewTableCell("Delete").
+			SetTextColor(tcell.ColorWhite).
+			SetAlign(tview.AlignCenter).
+			SetSelectable(false))
 	// Add the file rows starting from row 1
 	for r := 0; r < rows; r++ {
+		f := ragFiles[r]
 		for c := 0; c < cols; c++ {
 			color := tcell.ColorWhite
-			switch {
+			switch c {
-			case c < 1:
+			case 0:
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
+				displayName := f.name
-					tview.NewTableCell(fileList[r]).
+				if !f.inRAGDir {
+					displayName = f.name + " (orphaned)"
+				}
+				fileTable.SetCell(r+1, c,
+					tview.NewTableCell(displayName).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter).
 						SetSelectable(false))
-			case c == 1: // Action description column - not selectable
+			case 1:
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
+				if !f.inRAGDir {
-					tview.NewTableCell("(Action)").
+					// Orphaned file - no preview available
-						SetTextColor(color).
+					fileTable.SetCell(r+1, c,
-						SetAlign(tview.AlignCenter).
+						tview.NewTableCell("not in ragdir").
-						SetSelectable(false))
+							SetTextColor(tcell.ColorYellow).
-			default: // Action button column - selectable
+							SetAlign(tview.AlignCenter).
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
+							SetSelectable(false))
-					tview.NewTableCell(actions[c-1]).
+				} else if fi, err := os.Stat(f.fullPath); err == nil {
+					size := fi.Size()
+					modTime := fi.ModTime()
+					preview := fmt.Sprintf("%s | %s", formatSize(size), modTime.Format("2006-01-02 15:04"))
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell(preview).
+							SetTextColor(color).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				} else {
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell("error").
+							SetTextColor(color).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				}
+			case 2:
+				actionText := "load"
+				if f.isLoaded {
+					actionText = "unload"
+				}
+				if !f.inRAGDir {
+					// Orphaned file - can only unload
+					actionText = "unload"
+				}
+				fileTable.SetCell(r+1, c,
+					tview.NewTableCell(actionText).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter))
+			case 3:
+				if !f.inRAGDir {
+					// Orphaned file - cannot delete from ragdir (not there)
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell("-").
+							SetTextColor(tcell.ColorDarkGray).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				} else {
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell("delete").
+							SetTextColor(color).
+							SetAlign(tview.AlignCenter))
+				}
 			}
 		}
 	}
@@ -318,7 +419,7 @@ func makeRAGTable(fileList []string) *tview.Flex {
 	}()
 	fileTable.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') || key == tcell.KeyCtrlX {
@@ -335,30 +436,58 @@ func makeRAGTable(fileList []string) *tview.Flex {
 		}
 		// defer pages.RemovePage(RAGPage)
 		tc := fileTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		fileTable.SetSelectable(false, false)
 		// Check if the selected row is the exit row (row 0) - do this first to avoid index issues
 		if row == 0 {
 			pages.RemovePage(RAGPage)
 			return
 		}
-		// For file rows, get the filename (row index - 1 because of the exit row at index 0)
+		// For file rows, get the file info (row index - 1 because of the exit row at index 0)
-		fpath := fileList[row-1] // -1 to account for the exit row at index 0
+		f := ragFiles[row-1]
-		// notification := fmt.Sprintf("chat: %s; action: %s", fpath, tc.Text)
+		// Handle "-" case (orphaned file with no delete option)
+		if tc.Text == "-" {
+			pages.RemovePage(RAGPage)
+			return
+		}
 		switch tc.Text {
 		case "load":
-			fpath = path.Join(cfg.RAGDir, fpath)
+			fpath := path.Join(cfg.RAGDir, f.name)
 			longStatusView.SetText("clicked load")
 			go func() {
 				if err := ragger.LoadRAG(fpath); err != nil {
 					logger.Error("failed to embed file", "chat", fpath, "error", err)
 					_ = notifyUser("RAG", "failed to embed file; error: "+err.Error())
-					errCh <- err
+					app.QueueUpdate(func() {
-					// pages.RemovePage(RAGPage)
+						pages.RemovePage(RAGPage)
+					})
 					return
 				}
+				_ = notifyUser("RAG", "file loaded successfully")
+				app.QueueUpdate(func() {
+					pages.RemovePage(RAGPage)
+				})
+			}()
+			return
+		case "unload":
+			longStatusView.SetText("clicked unload")
+			go func() {
+				if err := ragger.RemoveFile(f.name); err != nil {
+					logger.Error("failed to unload file from RAG", "filename", f.name, "error", err)
+					_ = notifyUser("RAG", "failed to unload file; error: "+err.Error())
+					app.QueueUpdate(func() {
+						pages.RemovePage(RAGPage)
+					})
+					return
+				}
+				_ = notifyUser("RAG", "file unloaded successfully")
+				app.QueueUpdate(func() {
+					pages.RemovePage(RAGPage)
+				})
 			}()
 			return
 		case "delete":
-			fpath = path.Join(cfg.RAGDir, fpath)
+			fpath := path.Join(cfg.RAGDir, f.name)
 			if err := os.Remove(fpath); err != nil {
 				logger.Error("failed to delete file", "filename", fpath, "error", err)
 				return
@@ -383,114 +512,6 @@ func makeRAGTable(fileList []string) *tview.Flex {
 	return ragflex
 }
 
-func makeLoadedRAGTable(fileList []string) *tview.Flex {
-	actions := []string{"delete"}
-	rows, cols := len(fileList), len(actions)+1
-	// Add 1 extra row for the "exit" option at the top
-	fileTable := tview.NewTable().
-		SetBorders(true)
-	longStatusView := tview.NewTextView()
-	longStatusView.SetText("Loaded RAG files list")
-	longStatusView.SetBorder(true).SetTitle("status")
-	longStatusView.SetChangedFunc(func() {
-		app.Draw()
-	})
-	ragflex := tview.NewFlex().SetDirection(tview.FlexRow).
-		AddItem(longStatusView, 0, 10, false).
-		AddItem(fileTable, 0, 60, true)
-	// Add the exit option as the first row (row 0)
-	fileTable.SetCell(0, 0,
-		tview.NewTableCell("Exit Loaded Files manager").
-			SetTextColor(tcell.ColorWhite).
-			SetAlign(tview.AlignCenter).
-			SetSelectable(false))
-	fileTable.SetCell(0, 1,
-		tview.NewTableCell("(Close without action)").
-			SetTextColor(tcell.ColorGray).
-			SetAlign(tview.AlignCenter).
-			SetSelectable(false))
-	fileTable.SetCell(0, 2,
-		tview.NewTableCell("exit").
-			SetTextColor(tcell.ColorGray).
-			SetAlign(tview.AlignCenter))
-	// Add the file rows starting from row 1
-	for r := 0; r < rows; r++ {
-		for c := 0; c < cols; c++ {
-			color := tcell.ColorWhite
-			switch {
-			case c < 1:
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell(fileList[r]).
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter).
-						SetSelectable(false))
-			case c == 1: // Action description column - not selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell("(Action)").
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter).
-						SetSelectable(false))
-			default: // Action button column - selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell(actions[c-1]).
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter))
-			}
-		}
-	}
-	fileTable.Select(0, 0).
-		SetFixed(1, 1).
-		SetSelectable(true, false).
-		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
-		SetDoneFunc(func(key tcell.Key) {
-			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') || key == tcell.KeyCtrlX {
-				pages.RemovePage(RAGLoadedPage)
-				return
-			}
-		}).SetSelectedFunc(func(row int, column int) {
-		// If user selects a non-actionable column (0 or 1), move to first action column (2)
-		if column <= 1 {
-			if fileTable.GetColumnCount() > 2 {
-				fileTable.Select(row, 2) // Select first action column
-			}
-			return
-		}
-		tc := fileTable.GetCell(row, column)
-		// Check if the selected row is the exit row (row 0) - do this first to avoid index issues
-		if row == 0 {
-			pages.RemovePage(RAGLoadedPage)
-			return
-		}
-		// For file rows, get the filename (row index - 1 because of the exit row at index 0)
-		fpath := fileList[row-1] // -1 to account for the exit row at index 0
-		switch tc.Text {
-		case "delete":
-			if err := ragger.RemoveFile(fpath); err != nil {
-				logger.Error("failed to delete file from RAG", "filename", fpath, "error", err)
-				longStatusView.SetText(fmt.Sprintf("Error deleting file: %v", err))
-				return
-			}
-			if err := notifyUser("RAG file deleted", fpath+" was deleted from RAG system"); err != nil {
-				logger.Error("failed to send notification", "error", err)
-			}
-			longStatusView.SetText(fpath + " was deleted from RAG system")
-			return
-		default:
-			pages.RemovePage(RAGLoadedPage)
-			return
-		}
-	})
-	// Add input capture to the flex container to handle 'x' key for closing
-	ragflex.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
-		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
-			pages.RemovePage(RAGLoadedPage)
-			return nil
-		}
-		return event
-	})
-	return ragflex
-}
-
 func makeAgentTable(agentList []string) *tview.Table {
 	actions := []string{"filepath", "load"}
 	rows, cols := len(agentList), len(actions)+1
@@ -499,14 +520,14 @@ func makeAgentTable(agentList []string) *tview.Table {
 	for r := 0; r < rows; r++ {
 		for c := 0; c < cols; c++ {
 			color := tcell.ColorWhite
-			switch {
+			switch c {
-			case c < 1:
+			case 0:
 				chatActTable.SetCell(r, c,
 					tview.NewTableCell(agentList[r]).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter).
 						SetSelectable(false))
-			case c == 1:
+			case 1:
 				if actions[c-1] == "filepath" {
 					cc, ok := sysMap[agentList[r]]
 					if !ok {
@@ -533,7 +554,7 @@ func makeAgentTable(agentList []string) *tview.Table {
 	}
 	chatActTable.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') {
@@ -549,6 +570,8 @@ func makeAgentTable(agentList []string) *tview.Table {
 			return
 		}
 		tc := chatActTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		chatActTable.SetSelectable(false, false)
 		selected := agentList[row]
 		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
 		switch tc.Text {
@@ -630,7 +653,7 @@ func makeCodeBlockTable(codeBlocks []string) *tview.Table {
 	}
 	table.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') {
@@ -646,6 +669,8 @@ func makeCodeBlockTable(codeBlocks []string) *tview.Table {
 			return
 		}
 		tc := table.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		table.SetSelectable(false, false)
 		selected := codeBlocks[row]
 		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
 		switch tc.Text {
@@ -702,7 +727,7 @@ func makeImportChatTable(filenames []string) *tview.Table {
 	}
 	chatActTable.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') {
@@ -718,6 +743,8 @@ func makeImportChatTable(filenames []string) *tview.Table {
 			return
 		}
 		tc := chatActTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		chatActTable.SetSelectable(false, false)
 		selected := filenames[row]
 		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
 		switch tc.Text {
@@ -792,6 +819,7 @@ func makeFilePicker() *tview.Flex {
 	// --- NEW: search state ---
 	searching := false
 	searchQuery := ""
+	searchInputMode := false
 	// Helper function to check if a file has an allowed extension from config
 	hasAllowedExtension := func(filename string) bool {
 		if cfg.FilePickerExts == "" {
@@ -984,6 +1012,7 @@ func makeFilePicker() *tview.Flex {
 			case tcell.KeyEsc:
 				// Exit search, clear filter
 				searching = false
+				searchInputMode = false
 				searchQuery = ""
 				refreshList(currentDisplayDir, "")
 				return nil
@@ -993,16 +1022,80 @@ func makeFilePicker() *tview.Flex {
 					refreshList(currentDisplayDir, searchQuery)
 				}
 				return nil
-			case tcell.KeyRune:
+			case tcell.KeyEnter:
-				r := event.Rune()
+				// Exit search input mode and let normal processing handle selection
-				if r != 0 {
+				searchInputMode = false
-					searchQuery += string(r)
+				// Get the currently highlighted item in the list
-					refreshList(currentDisplayDir, searchQuery)
+				itemIndex := listView.GetCurrentItem()
+				if itemIndex >= 0 && itemIndex < listView.GetItemCount() {
+					itemText, _ := listView.GetItemText(itemIndex)
+					// Check for the exit option first
+					if strings.HasPrefix(itemText, "Exit file picker") {
+						pages.RemovePage(filePickerPage)
+						return nil
+					}
+					// Extract the actual filename/directory name by removing the type info
+					actualItemName := itemText
+					if bracketPos := strings.Index(itemText, " ["); bracketPos != -1 {
+						actualItemName = itemText[:bracketPos]
+					}
+					// Check if it's a directory (ends with /)
+					if strings.HasSuffix(actualItemName, "/") {
+						var targetDir string
+						if strings.HasPrefix(actualItemName, "../") {
+							// Parent directory
+							targetDir = path.Dir(currentDisplayDir)
+							if targetDir == currentDisplayDir && currentDisplayDir == "/" {
+								return nil
+							}
+						} else {
+							// Regular subdirectory
+							dirName := strings.TrimSuffix(actualItemName, "/")
+							targetDir = path.Join(currentDisplayDir, dirName)
+						}
+						// Navigate – clear search
+						if cfg.ImagePreview && imgPreview != nil {
+							imgPreview.SetImage(nil)
+						}
+						searching = false
+						searchInputMode = false
+						searchQuery = ""
+						refreshList(targetDir, "")
+						dirStack = append(dirStack, targetDir)
+						currentStackPos = len(dirStack) - 1
+						statusView.SetText("Current: " + targetDir)
+						return nil
+					} else {
+						// It's a file
+						filePath := path.Join(currentDisplayDir, actualItemName)
+						if info, err := os.Stat(filePath); err == nil && !info.IsDir() {
+							if isImageFile(actualItemName) {
+								SetImageAttachment(filePath)
+								statusView.SetText("Image attached: " + filePath + " (will be sent with next message)")
+								pages.RemovePage(filePickerPage)
+							} else {
+								textArea.SetText(filePath, true)
+								app.SetFocus(textArea)
+								pages.RemovePage(filePickerPage)
+							}
+						}
+						return nil
+					}
 				}
 				return nil
+			case tcell.KeyRune:
+				r := event.Rune()
+				if searchInputMode && r != 0 {
+					searchQuery += string(r)
+					refreshList(currentDisplayDir, searchQuery)
+					return nil
+				}
+				// If not in search input mode, pass through for navigation
+				return event
 			default:
-				// Pass all other keys (arrows, Enter, etc.) to normal processing
+				// Exit search input mode but keep filter active for navigation
-				// This allows selecting items while still in search mode
+				searchInputMode = false
+				// Pass all other keys (arrows, etc.) to normal processing
 				return event
 			}
 		}
@@ -1030,41 +1123,18 @@ func makeFilePicker() *tview.Flex {
 			if event.Rune() == '/' {
 				// Enter search mode
 				searching = true
+				searchInputMode = true
 				searchQuery = ""
 				refreshList(currentDisplayDir, "")
 				return nil
 			}
 			if event.Rune() == 's' {
 				// Set FilePickerDir to current directory
-				itemIndex := listView.GetCurrentItem()
+				// Get the actual directory path
-				if itemIndex >= 0 && itemIndex < listView.GetItemCount() {
+				cfg.FilePickerDir = currentDisplayDir
-					itemText, _ := listView.GetItemText(itemIndex)
+				listView.SetTitle("Files & Directories [s: set FilePickerDir]. Current base dir: " + cfg.FilePickerDir)
-					// Get the actual directory path
+				// pages.RemovePage(filePickerPage)
-					var targetDir string
+				return nil
-					if strings.HasPrefix(itemText, "Exit") || strings.HasPrefix(itemText, "Select this directory") {
-						targetDir = currentDisplayDir
-					} else {
-						actualItemName := itemText
-						if bracketPos := strings.Index(itemText, " ["); bracketPos != -1 {
-							actualItemName = itemText[:bracketPos]
-						}
-						// nolint: gocritic
-						if strings.HasPrefix(actualItemName, "../") {
-							targetDir = path.Dir(currentDisplayDir)
-						} else if strings.HasSuffix(actualItemName, "/") {
-							dirName := strings.TrimSuffix(actualItemName, "/")
-							targetDir = path.Join(currentDisplayDir, dirName)
-						} else {
-							targetDir = currentDisplayDir
-						}
-					}
-					cfg.FilePickerDir = targetDir
-					if err := notifyUser("FilePickerDir", "Set to: "+targetDir); err != nil {
-						logger.Error("failed to notify user", "error", err)
-					}
-					// pages.RemovePage(filePickerPage)
-					return nil
-				}
 			}
 		case tcell.KeyEnter:
 			// Get the currently highlighted item in the list

tools.go

@@ -16,6 +16,8 @@ import (
 	"sync"
 	"time"
 
+	"gf-lt/rag"
+
 	"github.com/GrailFinder/searchagent/searcher"
 )
 
@@ -58,9 +60,9 @@ Your current tools:
 "when_to_use": "when asked to search the web for information; returns clean summary without html,css and other web elements; limit is optional (default 3)"
 },
 {
-"name":"websearch_raw",
+"name":"rag_search",
 "args": ["query", "limit"],
-"when_to_use": "when asked to search the web for information; returns raw data as is without processing; limit is optional (default 3)"
+"when_to_use": "when asked to search the local document database for information; performs query refinement, semantic search, reranking, and synthesis; returns clean summary with sources; limit is optional (default 3)"
 },
 {
 "name":"read_url",
@@ -93,6 +95,11 @@ Your current tools:
 "when_to_use": "when asked to append content to a file; use sed to edit content"
 },
 {
+"name":"file_edit",
+"args": ["path", "oldString", "newString", "lineNumber"],
+"when_to_use": "when you need to make targeted changes to a specific section of a file without rewriting the entire file; lineNumber is optional - if provided, only edits that specific line; if not provided, replaces all occurrences of oldString"
+},
+{
 "name":"file_delete",
 "args": ["path"],
 "when_to_use": "when asked to delete a file"
@@ -115,7 +122,7 @@ Your current tools:
 {
 "name":"execute_command",
 "args": ["command", "args"],
-"when_to_use": "when asked to execute a system command; args is optional; allowed commands: grep, sed, awk, find, cat, head, tail, sort, uniq, wc, ls, echo, cut, tr, cp, mv, rm, mkdir, rmdir, pwd, df, free, ps, top, du, whoami, date, uname"
+"when_to_use": "when asked to execute a system command; args is optional; allowed commands: grep, sed, awk, find, cat, head, tail, sort, uniq, wc, ls, echo, cut, tr, cp, mv, rm, mkdir, rmdir, pwd, df, free, ps, top, du, whoami, date, uname, go"
 }
 ]
 </tools>
@@ -146,6 +153,7 @@ under the topic: Adam's number is stored:
 After that you are free to respond to the user.
 `
 	webSearchSysPrompt = `Summarize the web search results, extracting key information and presenting a concise answer. Provide sources and URLs where relevant.`
+	ragSearchSysPrompt = `Synthesize the document search results, extracting key information and presenting a concise answer. Provide sources and document IDs where relevant.`
 	readURLSysPrompt   = `Extract and summarize the content from the webpage. Provide key information, main points, and any relevant details.`
 	summarySysPrompt   = `Please provide a concise summary of the following conversation. Focus on key points, decisions, and actions. Provide only the summary, no additional commentary.`
 	basicCard          = &models.CharCard{
@@ -170,6 +178,9 @@ func init() {
 		panic("failed to init seachagent; error: " + err.Error())
 	}
 	WebSearcher = sa
+	if err := rag.Init(cfg, logger, store); err != nil {
+		logger.Warn("failed to init rag; rag_search tool will not be available", "error", err)
+	}
 }
 
 // getWebAgentClient returns a singleton AgentClient for web agents.
@@ -196,6 +207,8 @@ func getWebAgentClient() *agent.AgentClient {
 func registerWebAgents() {
 	webAgentsOnce.Do(func() {
 		client := getWebAgentClient()
+		// Register rag_search agent
+		agent.Register("rag_search", agent.NewWebAgentB(client, ragSearchSysPrompt))
 		// Register websearch agent
 		agent.Register("websearch", agent.NewWebAgentB(client, webSearchSysPrompt))
 		// Register read_url agent
@@ -239,6 +252,45 @@ func websearch(args map[string]string) []byte {
 	return data
 }
 
+// rag search (searches local document database)
+func ragsearch(args map[string]string) []byte {
+	query, ok := args["query"]
+	if !ok || query == "" {
+		msg := "query not provided to rag_search tool"
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	limitS, ok := args["limit"]
+	if !ok || limitS == "" {
+		limitS = "3"
+	}
+	limit, err := strconv.Atoi(limitS)
+	if err != nil || limit == 0 {
+		logger.Warn("ragsearch limit; passed bad value; setting to default (3)",
+			"limit_arg", limitS, "error", err)
+		limit = 3
+	}
+	ragInstance := rag.GetInstance()
+	if ragInstance == nil {
+		msg := "rag not initialized; rag_search tool is not available"
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	results, err := ragInstance.Search(query, limit)
+	if err != nil {
+		msg := "rag search failed; error: " + err.Error()
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	data, err := json.Marshal(results)
+	if err != nil {
+		msg := "failed to marshal rag search result; error: " + err.Error()
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	return data
+}
+
 // web search raw (returns raw data without processing)
 func websearchRaw(args map[string]string) []byte {
 	// make http request return bytes
@@ -369,7 +421,6 @@ func recallTopics(args map[string]string) []byte {
 }
 
 // File Manipulation Tools
-
 func fileCreate(args map[string]string) []byte {
 	path, ok := args["path"]
 	if !ok || path == "" {
@@ -377,20 +428,16 @@ func fileCreate(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	path = resolvePath(path)
-
 	content, ok := args["content"]
 	if !ok {
 		content = ""
 	}
-
 	if err := writeStringToFile(path, content); err != nil {
 		msg := "failed to create file; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	msg := "file created successfully at " + path
 	return []byte(msg)
 }
@@ -402,16 +449,13 @@ func fileRead(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	path = resolvePath(path)
-
 	content, err := readStringFromFile(path)
 	if err != nil {
 		msg := "failed to read file; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	result := map[string]string{
 		"content": content,
 		"path":    path,
@@ -422,7 +466,6 @@ func fileRead(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	return jsonResult
 }
 
@@ -468,6 +511,85 @@ func fileWriteAppend(args map[string]string) []byte {
 	return []byte(msg)
 }
 
+func fileEdit(args map[string]string) []byte {
+	path, ok := args["path"]
+	if !ok || path == "" {
+		msg := "path not provided to file_edit tool"
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	path = resolvePath(path)
+
+	oldString, ok := args["oldString"]
+	if !ok || oldString == "" {
+		msg := "oldString not provided to file_edit tool"
+		logger.Error(msg)
+		return []byte(msg)
+	}
+
+	newString, ok := args["newString"]
+	if !ok {
+		newString = ""
+	}
+
+	lineNumberStr, hasLineNumber := args["lineNumber"]
+
+	// Read file content
+	content, err := os.ReadFile(path)
+	if err != nil {
+		msg := "failed to read file: " + err.Error()
+		logger.Error(msg)
+		return []byte(msg)
+	}
+
+	fileContent := string(content)
+	var replacementCount int
+
+	if hasLineNumber && lineNumberStr != "" {
+		// Line-number based edit
+		lineNum, err := strconv.Atoi(lineNumberStr)
+		if err != nil {
+			msg := "invalid lineNumber: must be a valid integer"
+			logger.Error(msg)
+			return []byte(msg)
+		}
+		lines := strings.Split(fileContent, "\n")
+		if lineNum < 1 || lineNum > len(lines) {
+			msg := fmt.Sprintf("lineNumber %d out of range (file has %d lines)", lineNum, len(lines))
+			logger.Error(msg)
+			return []byte(msg)
+		}
+		// Find oldString in the specific line
+		targetLine := lines[lineNum-1]
+		if !strings.Contains(targetLine, oldString) {
+			msg := fmt.Sprintf("oldString not found on line %d", lineNum)
+			logger.Error(msg)
+			return []byte(msg)
+		}
+		lines[lineNum-1] = strings.Replace(targetLine, oldString, newString, 1)
+		replacementCount = 1
+		fileContent = strings.Join(lines, "\n")
+	} else {
+		// Replace all occurrences
+		if !strings.Contains(fileContent, oldString) {
+			msg := "oldString not found in file"
+			logger.Error(msg)
+			return []byte(msg)
+		}
+		fileContent = strings.ReplaceAll(fileContent, oldString, newString)
+		replacementCount = strings.Count(fileContent, newString)
+	}
+
+	if err := os.WriteFile(path, []byte(fileContent), 0644); err != nil {
+		msg := "failed to write file: " + err.Error()
+		logger.Error(msg)
+		return []byte(msg)
+	}
+
+	msg := fmt.Sprintf("file edited successfully at %s (%d replacement(s))", path, replacementCount)
+	return []byte(msg)
+}
+
 func fileDelete(args map[string]string) []byte {
 	path, ok := args["path"]
 	if !ok || path == "" {
@@ -475,15 +597,12 @@ func fileDelete(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	path = resolvePath(path)
-
 	if err := removeFile(path); err != nil {
 		msg := "failed to delete file; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	msg := "file deleted successfully at " + path
 	return []byte(msg)
 }
@@ -496,7 +615,6 @@ func fileMove(args map[string]string) []byte {
 		return []byte(msg)
 	}
 	src = resolvePath(src)
-
 	dst, ok := args["dst"]
 	if !ok || dst == "" {
 		msg := "destination path not provided to file_move tool"
@@ -504,13 +622,11 @@ func fileMove(args map[string]string) []byte {
 		return []byte(msg)
 	}
 	dst = resolvePath(dst)
-
 	if err := moveFile(src, dst); err != nil {
 		msg := "failed to move file; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	msg := fmt.Sprintf("file moved successfully from %s to %s", src, dst)
 	return []byte(msg)
 }
@@ -523,7 +639,6 @@ func fileCopy(args map[string]string) []byte {
 		return []byte(msg)
 	}
 	src = resolvePath(src)
-
 	dst, ok := args["dst"]
 	if !ok || dst == "" {
 		msg := "destination path not provided to file_copy tool"
@@ -531,13 +646,11 @@ func fileCopy(args map[string]string) []byte {
 		return []byte(msg)
 	}
 	dst = resolvePath(dst)
-
 	if err := copyFile(src, dst); err != nil {
 		msg := "failed to copy file; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	msg := fmt.Sprintf("file copied successfully from %s to %s", src, dst)
 	return []byte(msg)
 }
@@ -547,16 +660,13 @@ func fileList(args map[string]string) []byte {
 	if !ok || path == "" {
 		path = "." // default to current directory
 	}
-
 	path = resolvePath(path)
-
 	files, err := listDirectory(path)
 	if err != nil {
 		msg := "failed to list directory; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	result := map[string]interface{}{
 		"directory": path,
 		"files":     files,
@@ -567,12 +677,10 @@ func fileList(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	return jsonResult
 }
 
 // Helper functions for file operations
-
 func resolvePath(p string) string {
 	if filepath.IsAbs(p) {
 		return p
@@ -598,7 +706,6 @@ func appendStringToFile(filename string, data string) error {
 		return err
 	}
 	defer file.Close()
-
 	_, err = file.WriteString(data)
 	return err
 }
@@ -622,13 +729,11 @@ func copyFile(src, dst string) error {
 		return err
 	}
 	defer srcFile.Close()
-
 	dstFile, err := os.Create(dst)
 	if err != nil {
 		return err
 	}
 	defer dstFile.Close()
-
 	_, err = io.Copy(dstFile, srcFile)
 	return err
 }
@@ -647,7 +752,6 @@ func listDirectory(path string) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	var files []string
 	for _, entry := range entries {
 		if entry.IsDir() {
@@ -656,12 +760,10 @@ func listDirectory(path string) ([]string, error) {
 			files = append(files, entry.Name())
 		}
 	}
-
 	return files, nil
 }
 
 // Command Execution Tool
-
 func executeCommand(args map[string]string) []byte {
 	command, ok := args["command"]
 	if !ok || command == "" {
@@ -669,7 +771,6 @@ func executeCommand(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	// Get arguments - handle both single arg and multiple args
 	var cmdArgs []string
 	if args["args"] != "" {
@@ -688,47 +789,59 @@ func executeCommand(args map[string]string) []byte {
 			argNum++
 		}
 	}
-
+	// Handle commands passed as single string with spaces (e.g., "go run main.go")
+	// Split into base command and arguments
+	if strings.Contains(command, " ") {
+		parts := strings.Fields(command)
+		baseCmd := parts[0]
+		extraArgs := parts[1:]
+		// Prepend extra args to cmdArgs
+		cmdArgs = append(extraArgs, cmdArgs...)
+		command = baseCmd
+	}
 	if !isCommandAllowed(command, cmdArgs...) {
 		msg := fmt.Sprintf("command '%s' is not allowed", command)
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	// Execute with timeout for safety
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	cmd := exec.CommandContext(ctx, command, cmdArgs...)
-
+	cmd.Dir = cfg.FilePickerDir
 	output, err := cmd.CombinedOutput()
 	if err != nil {
 		msg := fmt.Sprintf("command '%s' failed; error: %v; output: %s", command, err, string(output))
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	// Check if output is empty and return success message
 	if len(output) == 0 {
 		successMsg := fmt.Sprintf("command '%s %s' executed successfully and exited with code 0", command, strings.Join(cmdArgs, " "))
 		return []byte(successMsg)
 	}
-
 	return output
 }
 
 // Helper functions for command execution
-
 // Todo structure
 type TodoItem struct {
 	ID     string `json:"id"`
 	Task   string `json:"task"`
 	Status string `json:"status"` // "pending", "in_progress", "completed"
 }
-
 type TodoList struct {
 	Items []TodoItem `json:"items"`
 }
 
+func (t TodoList) ToString() string {
+	sb := strings.Builder{}
+	for i := range t.Items {
+		fmt.Fprintf(&sb, "\n[%s] %s. %s\n", t.Items[i].Status, t.Items[i].ID, t.Items[i].Task)
+	}
+	return sb.String()
+}
+
 // Global todo list storage
 var globalTodoList = TodoList{
 	Items: []TodoItem{},
@@ -742,69 +855,34 @@ func todoCreate(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	// Generate simple ID
 	id := fmt.Sprintf("todo_%d", len(globalTodoList.Items)+1)
-
 	newItem := TodoItem{
 		ID:     id,
 		Task:   task,
 		Status: "pending",
 	}
-
 	globalTodoList.Items = append(globalTodoList.Items, newItem)
-
 	result := map[string]string{
 		"message": "todo created successfully",
 		"id":      id,
 		"task":    task,
 		"status":  "pending",
+		"todos":   globalTodoList.ToString(),
 	}
-
 	jsonResult, err := json.Marshal(result)
 	if err != nil {
 		msg := "failed to marshal result; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	return jsonResult
 }
 
 func todoRead(args map[string]string) []byte {
-	id, ok := args["id"]
-	if ok && id != "" {
-		// Find specific todo by ID
-		for _, item := range globalTodoList.Items {
-			if item.ID == id {
-				result := map[string]interface{}{
-					"todo": item,
-				}
-				jsonResult, err := json.Marshal(result)
-				if err != nil {
-					msg := "failed to marshal result; error: " + err.Error()
-					logger.Error(msg)
-					return []byte(msg)
-				}
-				return jsonResult
-			}
-		}
-		// ID not found
-		result := map[string]string{
-			"error": fmt.Sprintf("todo with id %s not found", id),
-		}
-		jsonResult, err := json.Marshal(result)
-		if err != nil {
-			msg := "failed to marshal result; error: " + err.Error()
-			logger.Error(msg)
-			return []byte(msg)
-		}
-		return jsonResult
-	}
-
 	// Return all todos if no ID specified
 	result := map[string]interface{}{
-		"todos": globalTodoList.Items,
+		"todos": globalTodoList.ToString(),
 	}
 	jsonResult, err := json.Marshal(result)
 	if err != nil {
@@ -812,7 +890,6 @@ func todoRead(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	return jsonResult
 }
 
@@ -823,16 +900,13 @@ func todoUpdate(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	task, taskOk := args["task"]
 	status, statusOk := args["status"]
-
 	if !taskOk && !statusOk {
 		msg := "neither task nor status provided to todo_update tool"
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	// Find and update the todo
 	for i, item := range globalTodoList.Items {
 		if item.ID == id {
@@ -856,23 +930,20 @@ func todoUpdate(args map[string]string) []byte {
 					return jsonResult
 				}
 			}
-
 			result := map[string]string{
 				"message": "todo updated successfully",
 				"id":      id,
+				"todos":   globalTodoList.ToString(),
 			}
-
 			jsonResult, err := json.Marshal(result)
 			if err != nil {
 				msg := "failed to marshal result; error: " + err.Error()
 				logger.Error(msg)
 				return []byte(msg)
 			}
-
 			return jsonResult
 		}
 	}
-
 	// ID not found
 	result := map[string]string{
 		"error": fmt.Sprintf("todo with id %s not found", id),
@@ -893,29 +964,25 @@ func todoDelete(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	// Find and remove the todo
 	for i, item := range globalTodoList.Items {
 		if item.ID == id {
 			// Remove item from slice
 			globalTodoList.Items = append(globalTodoList.Items[:i], globalTodoList.Items[i+1:]...)
-
 			result := map[string]string{
 				"message": "todo deleted successfully",
 				"id":      id,
+				"todos":   globalTodoList.ToString(),
 			}
-
 			jsonResult, err := json.Marshal(result)
 			if err != nil {
 				msg := "failed to marshal result; error: " + err.Error()
 				logger.Error(msg)
 				return []byte(msg)
 			}
-
 			return jsonResult
 		}
 	}
-
 	// ID not found
 	result := map[string]string{
 		"error": fmt.Sprintf("todo with id %s not found", id),
@@ -972,13 +1039,18 @@ func isCommandAllowed(command string, args ...string) bool {
 		"date":   true,
 		"uname":  true,
 		"git":    true,
+		"go":     true,
 	}
-	if !allowedCommands[command] {
+	// Allow all go subcommands (go run, go mod tidy, go test, etc.)
-		return false
+	if strings.HasPrefix(command, "go ") && allowedCommands["go"] {
+		return true
 	}
 	if command == "git" && len(args) > 0 {
 		return gitReadSubcommands[args[0]]
 	}
+	if !allowedCommands[command] {
+		return false
+	}
 	return true
 }
 
@@ -997,6 +1069,7 @@ var fnMap = map[string]fnSig{
 	"recall":            recall,
 	"recall_topics":     recallTopics,
 	"memorise":          memorise,
+	"rag_search":        ragsearch,
 	"websearch":         websearch,
 	"websearch_raw":     websearchRaw,
 	"read_url":          readURL,
@@ -1005,6 +1078,7 @@ var fnMap = map[string]fnSig{
 	"file_read":         fileRead,
 	"file_write":        fileWrite,
 	"file_write_append": fileWriteAppend,
+	"file_edit":         fileEdit,
 	"file_delete":       fileDelete,
 	"file_move":         fileMove,
 	"file_copy":         fileCopy,
@@ -1033,6 +1107,28 @@ func callToolWithAgent(name string, args map[string]string) []byte {
 
 // openai style def
 var baseTools = []models.Tool{
+	// rag_search
+	models.Tool{
+		Type: "function",
+		Function: models.ToolFunc{
+			Name:        "rag_search",
+			Description: "Search local document database given query, limit of sources (default 3). Performs query refinement, semantic search, reranking, and synthesis.",
+			Parameters: models.ToolFuncParams{
+				Type:     "object",
+				Required: []string{"query", "limit"},
+				Properties: map[string]models.ToolArgProps{
+					"query": models.ToolArgProps{
+						Type:        "string",
+						Description: "search query",
+					},
+					"limit": models.ToolArgProps{
+						Type:        "string",
+						Description: "limit of the document results",
+					},
+				},
+			},
+		},
+	},
 	// websearch
 	models.Tool{
 		Type: "function",
@@ -1166,7 +1262,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_create
 	models.Tool{
 		Type: "function",
@@ -1189,7 +1284,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_read
 	models.Tool{
 		Type: "function",
@@ -1208,7 +1302,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_write
 	models.Tool{
 		Type: "function",
@@ -1231,7 +1324,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_write_append
 	models.Tool{
 		Type: "function",
@@ -1254,7 +1346,36 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
+	// file_edit
+	models.Tool{
+		Type: "function",
+		Function: models.ToolFunc{
+			Name:        "file_edit",
+			Description: "Edit a specific section of a file by replacing oldString with newString. Use for targeted changes without rewriting the entire file.",
+			Parameters: models.ToolFuncParams{
+				Type:     "object",
+				Required: []string{"path", "oldString", "newString"},
+				Properties: map[string]models.ToolArgProps{
+					"path": models.ToolArgProps{
+						Type:        "string",
+						Description: "path of the file to edit",
+					},
+					"oldString": models.ToolArgProps{
+						Type:        "string",
+						Description: "the exact string to find and replace",
+					},
+					"newString": models.ToolArgProps{
+						Type:        "string",
+						Description: "the string to replace oldString with",
+					},
+					"lineNumber": models.ToolArgProps{
+						Type:        "string",
+						Description: "optional line number (1-indexed) to edit - if provided, only that line is edited",
+					},
+				},
+			},
+		},
+	},
 	// file_delete
 	models.Tool{
 		Type: "function",
@@ -1273,7 +1394,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_move
 	models.Tool{
 		Type: "function",
@@ -1296,7 +1416,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_copy
 	models.Tool{
 		Type: "function",
@@ -1319,7 +1438,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_list
 	models.Tool{
 		Type: "function",
@@ -1338,20 +1456,19 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// execute_command
 	models.Tool{
 		Type: "function",
 		Function: models.ToolFunc{
 			Name:        "execute_command",
-			Description: "Execute a shell command safely. Use when you need to run system commands like grep sed awk find cat head tail sort uniq wc ls echo cut tr cp mv rm mkdir rmdir pwd df free ps top du whoami date uname. Git is allowed for read-only operations: status, log, diff, show, branch, reflog, rev-parse, shortlog, describe.",
+			Description: "Execute a shell command safely. Use when you need to run system commands like grep sed awk find cat head tail sort uniq wc ls echo cut tr cp mv rm mkdir rmdir pwd df free ps top du whoami date uname go. Git is allowed for read-only operations: status, log, diff, show, branch, reflog, rev-parse, shortlog, describe.",
 			Parameters: models.ToolFuncParams{
 				Type:     "object",
 				Required: []string{"command"},
 				Properties: map[string]models.ToolArgProps{
 					"command": models.ToolArgProps{
 						Type:        "string",
-						Description: "command to execute (only commands from whitelist are allowed: grep sed awk find cat head tail sort uniq wc ls echo cut tr cp mv rm mkdir rmdir pwd df free ps top du whoami date uname; git allowed for reads: status log diff show branch reflog rev-parse shortlog describe)",
+						Description: "command to execute (only commands from whitelist are allowed: grep sed awk find cat head tail sort uniq wc ls echo cut tr cp mv rm mkdir rmdir pwd df free ps top du whoami date uname go; git allowed for reads: status log diff show branch reflog rev-parse shortlog describe)",
 					},
 					"args": models.ToolArgProps{
 						Type:        "string",
@@ -1404,22 +1521,9 @@ var baseTools = []models.Tool{
 			Name:        "todo_update",
 			Description: "Update a todo item by ID with new task or status. Status must be one of: pending, in_progress, completed.",
 			Parameters: models.ToolFuncParams{
-				Type:     "object",
+				Type:       "object",
-				Required: []string{"id"},
+				Required:   []string{},
-				Properties: map[string]models.ToolArgProps{
+				Properties: map[string]models.ToolArgProps{},
-					"id": models.ToolArgProps{
-						Type:        "string",
-						Description: "id of the todo item to update",
-					},
-					"task": models.ToolArgProps{
-						Type:        "string",
-						Description: "new task description (optional)",
-					},
-					"status": models.ToolArgProps{
-						Type:        "string",
-						Description: "new status for the todo: pending, in_progress, or completed (optional)",
-					},
-				},
 			},
 		},
 	},

tui.go

@@ -15,6 +15,11 @@ import (
 	"github.com/rivo/tview"
 )
 
+func isFullScreenPageActive() bool {
+	name, _ := pages.GetFrontPage()
+	return name != "main"
+}
+
 var (
 	app                *tview.Application
 	pages              *tview.Pages
@@ -29,6 +34,7 @@ var (
 	indexPickWindow    *tview.InputField
 	renameWindow       *tview.InputField
 	roleEditWindow     *tview.InputField
+	shellInput         *tview.InputField
 	fullscreenMode     bool
 	positionVisible    bool = true
 	scrollToEndEnabled bool = true
@@ -74,7 +80,7 @@ var (
 [yellow]Ctrl+p[white]: props edit form (min-p, dry, etc.)
 [yellow]Ctrl+v[white]: show API link selection popup to choose current API
 [yellow]Ctrl+r[white]: start/stop recording from your microphone (needs stt server or whisper binary)
-[yellow]Ctrl+t[white]: remove thinking (<think>) and tool messages from context (delete from chat)
+[yellow]Ctrl+t[white]: (un)collapse tool messages
 [yellow]Ctrl+l[white]: show model selection popup to choose current model
 [yellow]Ctrl+k[white]: switch tool use (recommend tool use to llm after user msg)
 [yellow]Ctrl+a[white]: interrupt tts (needs tts server)
@@ -93,6 +99,7 @@ var (
 [yellow]Alt+8[white]: show char img or last picked img
 [yellow]Alt+9[white]: warm up (load) selected llama.cpp model
 [yellow]Alt+t[white]: toggle thinking blocks visibility (collapse/expand <think> blocks)
+[yellow]Ctrl+t[white]: toggle tool call/response visibility (collapse/expand tool calls and non-shell tool responses)
 [yellow]Alt+i[white]: show colorscheme selection popup
 
 === scrolling chat window (some keys similar to vim) ===
@@ -119,46 +126,78 @@ Press <Enter> or 'x' to return
 `
 )
 
+func setShellMode(enabled bool) {
+	shellMode = enabled
+	go func() {
+		app.QueueUpdateDraw(func() {
+			updateFlexLayout()
+		})
+	}()
+}
+
 func init() {
 	// Start background goroutine to update model color cache
 	startModelColorUpdater()
 	tview.Styles = colorschemes["default"]
 	app = tview.NewApplication()
 	pages = tview.NewPages()
-	textArea = tview.NewTextArea().
+	shellInput = tview.NewInputField().
-		SetPlaceholder("input is multiline; press <Enter> to start the next line;\npress <Esc> to send the message.")
+		SetLabel(fmt.Sprintf("[%s]$ ", cfg.FilePickerDir)). // dynamic prompt
-	textArea.SetBorder(true).SetTitle("input")
+		SetFieldWidth(0).
-	// Add input capture for @ completion
+		SetDoneFunc(func(key tcell.Key) {
-	textArea.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+			if key == tcell.KeyEnter {
+				cmd := shellInput.GetText()
+				if cmd != "" {
+					executeCommandAndDisplay(cmd)
+				}
+				shellInput.SetText("")
+			}
+		})
+	// Copy your file completion logic to shellInput's InputCapture
+	shellInput.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if !shellMode {
 			return event
 		}
-		// Handle Tab key for file completion
+		// Handle Up arrow for history previous
-		if event.Key() == tcell.KeyTab {
+		if event.Key() == tcell.KeyUp {
-			currentText := textArea.GetText()
+			if len(shellHistory) > 0 {
-			row, col, _, _ := textArea.GetCursor()
+				if shellHistoryPos < len(shellHistory)-1 {
-			// Calculate absolute position from row/col
+					shellHistoryPos++
-			lines := strings.Split(currentText, "\n")
+					shellInput.SetText(shellHistory[len(shellHistory)-1-shellHistoryPos])
-			cursorPos := 0
-			for i := 0; i < row && i < len(lines); i++ {
-				cursorPos += len(lines[i]) + 1 // +1 for newline
-			}
-			cursorPos += col
-			// Look backwards from cursor to find @
-			if cursorPos > 0 {
-				// Find the last @ before cursor
-				textBeforeCursor := currentText[:cursorPos]
-				atIndex := strings.LastIndex(textBeforeCursor, "@")
-				if atIndex >= 0 {
-					// Extract the partial match text after @
-					filter := textBeforeCursor[atIndex+1:]
-					showFileCompletionPopup(filter)
-					return nil // Consume the Tab event
 				}
 			}
+			return nil
+		}
+		// Handle Down arrow for history next
+		if event.Key() == tcell.KeyDown {
+			if shellHistoryPos > 0 {
+				shellHistoryPos--
+				shellInput.SetText(shellHistory[len(shellHistory)-1-shellHistoryPos])
+			} else if shellHistoryPos == 0 {
+				shellHistoryPos = -1
+				shellInput.SetText("")
+			}
+			return nil
+		}
+		// Reset history position when user types
+		if event.Key() == tcell.KeyRune {
+			shellHistoryPos = -1
+		}
+		// Handle Tab key for @ file completion
+		if event.Key() == tcell.KeyTab {
+			currentText := shellInput.GetText()
+			atIndex := strings.LastIndex(currentText, "@")
+			if atIndex >= 0 {
+				filter := currentText[atIndex+1:]
+				showShellFileCompletionPopup(filter)
+			}
+			return nil
 		}
 		return event
 	})
+	textArea = tview.NewTextArea().
+		SetPlaceholder("input is multiline; press <Enter> to start the next line;\npress <Esc> to send the message.")
+	textArea.SetBorder(true).SetTitle("input")
 	textView = tview.NewTextView().
 		SetDynamicColors(true).
 		SetRegions(true).
@@ -259,7 +298,7 @@ func init() {
 				pages.RemovePage(editMsgPage)
 				return nil
 			}
-			chatBody.Messages[selectedIndex].Content = editedMsg
+			chatBody.Messages[selectedIndex].SetText(editedMsg)
 			// change textarea
 			textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
 			pages.RemovePage(editMsgPage)
@@ -347,13 +386,14 @@ func init() {
 			case editMode:
 				hideIndexBar() // Hide overlay first
 				pages.AddPage(editMsgPage, editArea, true, true)
-				editArea.SetText(m.Content, true)
+				editArea.SetText(m.GetText(), true)
 			default:
-				if err := copyToClipboard(m.Content); err != nil {
+				msgText := m.GetText()
+				if err := copyToClipboard(msgText); err != nil {
 					logger.Error("failed to copy to clipboard", "error", err)
 				}
-				previewLen := min(30, len(m.Content))
+				previewLen := min(30, len(msgText))
-				notification := fmt.Sprintf("msg '%s' was copied to the clipboard", m.Content[:previewLen])
+				notification := fmt.Sprintf("msg '%s' was copied to the clipboard", msgText[:previewLen])
 				if err := notifyUser("copied", notification); err != nil {
 					logger.Error("failed to send notification", "error", err)
 				}
@@ -524,7 +564,24 @@ func init() {
 			}
 			return nil
 		}
+		// Handle Ctrl+T to toggle tool call/response visibility
+		if event.Key() == tcell.KeyCtrlT {
+			toolCollapsed = !toolCollapsed
+			textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
+			colorText()
+			status := "expanded"
+			if toolCollapsed {
+				status = "collapsed"
+			}
+			if err := notifyUser("tools", "Tool calls/responses "+status); err != nil {
+				logger.Error("failed to send notification", "error", err)
+			}
+			return nil
+		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'i' && event.Modifiers()&tcell.ModAlt != 0 {
+			if isFullScreenPageActive() {
+				return event
+			}
 			showColorschemeSelectionPopup()
 			return nil
 		}
@@ -640,11 +697,12 @@ func init() {
 			// copy msg to clipboard
 			editMode = false
 			m := chatBody.Messages[len(chatBody.Messages)-1]
-			if err := copyToClipboard(m.Content); err != nil {
+			msgText := m.GetText()
+			if err := copyToClipboard(msgText); err != nil {
 				logger.Error("failed to copy to clipboard", "error", err)
 			}
-			previewLen := min(30, len(m.Content))
+			previewLen := min(30, len(msgText))
-			notification := fmt.Sprintf("msg '%s' was copied to the clipboard", m.Content[:previewLen])
+			notification := fmt.Sprintf("msg '%s' was copied to the clipboard", msgText[:previewLen])
 			if err := notifyUser("copied", notification); err != nil {
 				logger.Error("failed to send notification", "error", err)
 			}
@@ -731,19 +789,17 @@ func init() {
 			return nil
 		}
 		if event.Key() == tcell.KeyCtrlL {
+			if isFullScreenPageActive() {
+				return event
+			}
 			// Show model selection popup instead of rotating models
 			showModelSelectionPopup()
 			return nil
 		}
-		if event.Key() == tcell.KeyCtrlT {
-			// clear context
-			// remove tools and thinking
-			removeThinking(chatBody)
-			textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
-			colorText()
-			return nil
-		}
 		if event.Key() == tcell.KeyCtrlV {
+			if isFullScreenPageActive() {
+				return event
+			}
 			// Show API link selection popup instead of rotating APIs
 			showAPILinkSelectionPopup()
 			return nil
@@ -833,7 +889,7 @@ func init() {
 				// Stop any currently playing TTS first
 				TTSDoneChan <- true
 				lastMsg := chatBody.Messages[len(chatBody.Messages)-1]
-				cleanedText := models.CleanText(lastMsg.Content)
+				cleanedText := models.CleanText(lastMsg.GetText())
 				if cleanedText != "" {
 					// nolint: errcheck
 					go orator.Speak(cleanedText)
@@ -850,11 +906,17 @@ func init() {
 			return nil
 		}
 		if event.Key() == tcell.KeyCtrlQ {
+			if isFullScreenPageActive() {
+				return event
+			}
 			// Show user role selection popup instead of cycling through roles
 			showUserRoleSelectionPopup()
 			return nil
 		}
 		if event.Key() == tcell.KeyCtrlX {
+			if isFullScreenPageActive() {
+				return event
+			}
 			// Show bot role selection popup instead of cycling through roles
 			showBotRoleSelectionPopup()
 			return nil
@@ -893,6 +955,7 @@ func init() {
 					return nil
 				}
 			}
+			// Get files from ragdir
 			fileList := []string{}
 			for _, f := range files {
 				if f.IsDir() {
@@ -900,22 +963,14 @@ func init() {
 				}
 				fileList = append(fileList, f.Name())
 			}
-			chatRAGTable := makeRAGTable(fileList)
+			// Get loaded files from vector DB
-			pages.AddPage(RAGPage, chatRAGTable, true, true)
+			loadedFiles, err := ragger.ListLoaded()
-			return nil
-		}
-		if event.Key() == tcell.KeyCtrlY { // Use Ctrl+Y to list loaded RAG files
-			// List files already loaded into the RAG system
-			fileList, err := ragger.ListLoaded()
 			if err != nil {
 				logger.Error("failed to list loaded RAG files", "error", err)
-				if notifyerr := notifyUser("failed to list RAG files", err.Error()); notifyerr != nil {
+				loadedFiles = []string{} // Continue with empty list on error
-					logger.Error("failed to send notification", "error", notifyerr)
-				}
-				return nil
 			}
-			chatLoadedRAGTable := makeLoadedRAGTable(fileList)
+			chatRAGTable := makeRAGTable(fileList, loadedFiles)
-			pages.AddPage(RAGLoadedPage, chatLoadedRAGTable, true, true)
+			pages.AddPage(RAGPage, chatRAGTable, true, true)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Modifiers() == tcell.ModAlt && event.Rune() == '1' {
@@ -933,14 +988,16 @@ func init() {
 		}
 		// cannot send msg in editMode or botRespMode
 		if event.Key() == tcell.KeyEscape && !editMode && !botRespMode {
-			msgText := textArea.GetText()
+			if shellMode {
-			if shellMode && msgText != "" {
+				cmdText := shellInput.GetText()
-				// In shell mode, execute command instead of sending to LLM
+				if cmdText != "" {
-				executeCommandAndDisplay(msgText)
+					executeCommandAndDisplay(cmdText)
-				textArea.SetText("", true) // Clear the input area
+					shellInput.SetText("")
+				}
 				return nil
-			} else if !shellMode {
+			}
-				// Normal mode - send to LLM
+			msgText := textArea.GetText()
+			if msgText != "" {
 				nl := "\n\n" // keep empty lines between messages
 				prevText := textView.GetText(true)
 				persona := cfg.UserRole
@@ -975,13 +1032,6 @@ func init() {
 				}
 				// go chatRound(msgText, persona, textView, false, false)
 				chatRoundChan <- &models.ChatRoundReq{Role: persona, UserMsg: msgText}
-				// Also clear any image attachment after sending the message
-				go func() {
-					// Wait a short moment for the message to be processed, then clear the image attachment
-					// This allows the image to be sent with the current message if it was attached
-					// But clears it for the next message
-					ClearImageAttachment()
-				}()
 			}
 			return nil
 		}