Enha: compute estimate of non llm text

logic requires reference to config

Makefile

@@ -1,4 +1,4 @@
-.PHONY: setconfig run lint setup-whisper build-whisper download-whisper-model docker-up docker-down docker-logs noextra-run
+.PHONY: setconfig run lint lintall install-linters setup-whisper build-whisper download-whisper-model docker-up docker-down docker-logs noextra-run installdelve checkdelve
 
 run: setconfig
 	go build -tags extra -o gf-lt && ./gf-lt
@@ -15,9 +15,21 @@ noextra-run: setconfig
 setconfig:
 	find config.toml &>/dev/null || cp config.example.toml config.toml
 
+installdelve:
+	go install github.com/go-delve/delve/cmd/dlv@latest
+
+checkdelve:
+	which dlv &>/dev/null || installdelve
+
+install-linters: ## Install additional linters (noblanks)
+	go install github.com/GrailFinder/noblanks-linter/cmd/noblanks@latest
+
 lint: ## Run linters. Use make install-linters first.
 	golangci-lint run -c .golangci.yml ./...
 
+lintall: lint
+	noblanks ./...
+
 # Whisper STT Setup (in batteries directory)
 setup-whisper: build-whisper download-whisper-model
 

agent/request.go

@@ -71,8 +71,8 @@ func (ag *AgentClient) buildRequest(sysprompt, msg string) ([]byte, error) {
 	// Build prompt for completion endpoints
 	if isCompletion {
 		var sb strings.Builder
-		for _, m := range messages {
-			sb.WriteString(m.ToPrompt())
+		for i := range messages {
+			sb.WriteString(messages[i].ToPrompt())
 			sb.WriteString("\n")
 		}
 		prompt := strings.TrimSpace(sb.String())
@@ -140,7 +140,6 @@ func (ag *AgentClient) LLMRequest(body io.Reader) ([]byte, error) {
 		ag.log.Error("failed to read request body", "error", err)
 		return nil, err
 	}
-
 	req, err := http.NewRequest("POST", ag.cfg.CurrentAPI, bytes.NewReader(bodyBytes))
 	if err != nil {
 		ag.log.Error("failed to create request", "error", err)
@@ -150,22 +149,18 @@ func (ag *AgentClient) LLMRequest(body io.Reader) ([]byte, error) {
 	req.Header.Add("Content-Type", "application/json")
 	req.Header.Add("Authorization", "Bearer "+ag.getToken())
 	req.Header.Set("Accept-Encoding", "gzip")
-
 	ag.log.Debug("agent LLM request", "url", ag.cfg.CurrentAPI, "body_preview", string(bodyBytes[:min(len(bodyBytes), 500)]))
-
 	resp, err := httpClient.Do(req)
 	if err != nil {
 		ag.log.Error("llamacpp api request failed", "error", err, "url", ag.cfg.CurrentAPI)
 		return nil, err
 	}
 	defer resp.Body.Close()
-
 	responseBytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		ag.log.Error("failed to read response", "error", err)
 		return nil, err
 	}
-
 	if resp.StatusCode >= 400 {
 		ag.log.Error("agent LLM request failed", "status", resp.StatusCode, "response", string(responseBytes[:min(len(responseBytes), 1000)]))
 		return responseBytes, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(responseBytes[:min(len(responseBytes), 200)]))
@@ -178,7 +173,6 @@ func (ag *AgentClient) LLMRequest(body io.Reader) ([]byte, error) {
 		// Return raw response as fallback
 		return responseBytes, nil
 	}
-
 	return []byte(text), nil
 }
 

bot.go

@@ -23,8 +23,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-
-	"github.com/neurosnap/sentences/english"
 )
 
 var (
@@ -65,9 +63,13 @@ var (
 		"google/gemma-3-27b-it:free",
 		"meta-llama/llama-3.3-70b-instruct:free",
 	}
-	LocalModels = []string{}
+	LocalModels     = []string{}
+	localModelsData *models.LCPModels
+	orModelsData    *models.ORModels
 )
 
+var thinkBlockRE = regexp.MustCompile(`(?s)<think>.*?</think>`)
+
 // parseKnownToTag extracts known_to list from content using configured tag.
 // Returns cleaned content and list of character names.
 func parseKnownToTag(content string) []string {
@@ -119,7 +121,7 @@ func processMessageTag(msg *models.RoleMsg) *models.RoleMsg {
 	}
 	// If KnownTo already set, assume tag already processed (content cleaned).
 	// However, we still check for new tags (maybe added later).
-	knownTo := parseKnownToTag(msg.Content)
+	knownTo := parseKnownToTag(msg.GetText())
 	// If tag found, replace KnownTo with new list (merge with existing?)
 	// For simplicity, if knownTo is not nil, replace.
 	if knownTo == nil {
@@ -138,6 +140,9 @@ func processMessageTag(msg *models.RoleMsg) *models.RoleMsg {
 // filterMessagesForCharacter returns messages visible to the specified character.
 // If CharSpecificContextEnabled is false, returns all messages.
 func filterMessagesForCharacter(messages []models.RoleMsg, character string) []models.RoleMsg {
+	if strings.Contains(cfg.CurrentAPI, "chat") {
+		return messages
+	}
 	if cfg == nil || !cfg.CharSpecificContextEnabled || character == "" {
 		return messages
 	}
@@ -145,97 +150,67 @@ func filterMessagesForCharacter(messages []models.RoleMsg, character string) []m
 		return messages
 	}
 	filtered := make([]models.RoleMsg, 0, len(messages))
-	for _, msg := range messages {
+	for i := range messages {
 		// If KnownTo is nil or empty, message is visible to all
 		// system msg cannot be filtered
-		if len(msg.KnownTo) == 0 || msg.Role == "system" {
-			filtered = append(filtered, msg)
+		if len(messages[i].KnownTo) == 0 || messages[i].Role == "system" {
+			filtered = append(filtered, messages[i])
 			continue
 		}
-		if slices.Contains(msg.KnownTo, character) {
+		if slices.Contains(messages[i].KnownTo, character) {
 			// Check if character is in KnownTo lis
-			filtered = append(filtered, msg)
+			filtered = append(filtered, messages[i])
 		}
 	}
 	return filtered
 }
 
-func cleanToolCalls(messages []models.RoleMsg) []models.RoleMsg {
-	// If AutoCleanToolCallsFromCtx is false, keep tool call messages in context
-	if cfg != nil && !cfg.AutoCleanToolCallsFromCtx {
-		return consolidateAssistantMessages(messages)
-	}
-	cleaned := make([]models.RoleMsg, 0, len(messages))
-	for i, msg := range messages {
-		// recognize the message as the tool call and remove it
-		// tool call in last msg should stay
-		if msg.ToolCallID == "" || i == len(messages)-1 {
-			cleaned = append(cleaned, msg)
-		}
-	}
-	return consolidateAssistantMessages(cleaned)
-}
-
-// consolidateAssistantMessages merges consecutive assistant messages into a single message
 func consolidateAssistantMessages(messages []models.RoleMsg) []models.RoleMsg {
 	if len(messages) == 0 {
 		return messages
 	}
-	consolidated := make([]models.RoleMsg, 0, len(messages))
-	currentAssistantMsg := models.RoleMsg{}
-	isBuildingAssistantMsg := false
-	for i := 0; i < len(messages); i++ {
-		msg := messages[i]
-		// assistant role only
-		if msg.Role == cfg.AssistantRole {
-			// If this is an assistant message, start or continue building
-			if !isBuildingAssistantMsg {
-				// Start accumulating assistant message
-				currentAssistantMsg = msg.Copy()
-				isBuildingAssistantMsg = true
-			} else {
-				// Continue accumulating - append content to the current assistant message
-				if currentAssistantMsg.IsContentParts() || msg.IsContentParts() {
-					// Handle structured content
-					if !currentAssistantMsg.IsContentParts() {
-						// Preserve the original ToolCallID before conversion
-						originalToolCallID := currentAssistantMsg.ToolCallID
-						// Convert existing content to content parts
-						currentAssistantMsg = models.NewMultimodalMsg(currentAssistantMsg.Role, []interface{}{models.TextContentPart{Type: "text", Text: currentAssistantMsg.Content}})
-						// Restore the original ToolCallID to preserve tool call linking
-						currentAssistantMsg.ToolCallID = originalToolCallID
-					}
-					if msg.IsContentParts() {
-						currentAssistantMsg.ContentParts = append(currentAssistantMsg.ContentParts, msg.GetContentParts()...)
-					} else if msg.Content != "" {
-						currentAssistantMsg.AddTextPart(msg.Content)
-					}
-				} else {
-					// Simple string content
-					if currentAssistantMsg.Content != "" {
-						currentAssistantMsg.Content += "\n" + msg.Content
-					} else {
-						currentAssistantMsg.Content = msg.Content
-					}
-					// ToolCallID is already preserved since we're not creating a new message object when just concatenating content
-				}
+	result := make([]models.RoleMsg, 0, len(messages))
+	for i := range messages {
+		// Non-assistant messages are appended as-is
+		if messages[i].Role != cfg.AssistantRole {
+			result = append(result, messages[i])
+			continue
+		}
+		// Assistant message: start a new block or merge with the last one
+		if len(result) == 0 || result[len(result)-1].Role != cfg.AssistantRole {
+			// First assistant in a block: append a copy (avoid mutating input)
+			result = append(result, messages[i].Copy())
+			continue
+		}
+		// Merge with the last assistant message
+		last := &result[len(result)-1]
+		// If either message has structured content, unify to ContentParts
+		if last.IsContentParts() || messages[i].IsContentParts() {
+			// Convert last to ContentParts if needed, preserving ToolCallID
+			if !last.IsContentParts() {
+				toolCallID := last.ToolCallID
+				*last = models.NewMultimodalMsg(last.Role, []interface{}{
+					models.TextContentPart{Type: "text", Text: last.Content},
+				})
+				last.ToolCallID = toolCallID
+			}
+			// Add current message's content to last
+			if messages[i].IsContentParts() {
+				last.ContentParts = append(last.ContentParts, messages[i].GetContentParts()...)
+			} else if messages[i].Content != "" {
+				last.AddTextPart(messages[i].Content)
 			}
 		} else {
-			// This is not an assistant message
-			// If we were building an assistant message, add it to the result
-			if isBuildingAssistantMsg {
-				consolidated = append(consolidated, currentAssistantMsg)
-				isBuildingAssistantMsg = false
+			// Both simple strings: concatenate with newline
+			if last.Content != "" && messages[i].Content != "" {
+				last.Content += "\n" + messages[i].Content
+			} else if messages[i].Content != "" {
+				last.Content = messages[i].Content
 			}
-			// Add the non-assistant message
-			consolidated = append(consolidated, msg)
+			// ToolCallID is already preserved in last
 		}
 	}
-	// Don't forget the last assistant message if we were building one
-	if isBuildingAssistantMsg {
-		consolidated = append(consolidated, currentAssistantMsg)
-	}
-	return consolidated
+	return result
 }
 
 // GetLogLevel returns the current log level as a string
@@ -382,6 +357,7 @@ func fetchORModels(free bool) ([]string, error) {
 	if err := json.NewDecoder(resp.Body).Decode(data); err != nil {
 		return nil, err
 	}
+	orModelsData = data
 	freeModels := data.ListModels(free)
 	return freeModels, nil
 }
@@ -406,19 +382,26 @@ func fetchLCPModels() ([]string, error) {
 
 // fetchLCPModelsWithLoadStatus returns models with "(loaded)" indicator for loaded models
 func fetchLCPModelsWithLoadStatus() ([]string, error) {
-	models, err := fetchLCPModelsWithStatus()
+	modelList, err := fetchLCPModelsWithStatus()
 	if err != nil {
 		return nil, err
 	}
-	result := make([]string, 0, len(models.Data))
-	for _, m := range models.Data {
+	result := make([]string, 0, len(modelList.Data))
+	li := 0 // loaded index
+	for i, m := range modelList.Data {
 		modelName := m.ID
 		if m.Status.Value == "loaded" {
-			modelName = "(loaded) " + modelName
+			modelName = models.LoadedMark + modelName
+			li = i
 		}
 		result = append(result, modelName)
 	}
-	return result, nil
+	if li == 0 {
+		return result, nil // no loaded modelList
+	}
+	loadedModel := result[li]
+	result = append(result[:li], result[li+1:]...)
+	return slices.Concat([]string{loadedModel}, result), nil
 }
 
 // fetchLCPModelsWithStatus returns the full LCPModels struct including status information.
@@ -436,6 +419,7 @@ func fetchLCPModelsWithStatus() (*models.LCPModels, error) {
 	if err := json.NewDecoder(resp.Body).Decode(data); err != nil {
 		return nil, err
 	}
+	localModelsData = data
 	return data, nil
 }
 
@@ -453,6 +437,33 @@ func isModelLoaded(modelID string) (bool, error) {
 	return false, nil
 }
 
+func ModelHasVision(api, modelID string) bool {
+	switch {
+	case strings.Contains(api, "deepseek"):
+		return false
+	case strings.Contains(api, "openrouter"):
+		resp, err := http.Get("https://openrouter.ai/api/v1/models")
+		if err != nil {
+			logger.Warn("failed to fetch OR models for vision check", "error", err)
+			return false
+		}
+		defer resp.Body.Close()
+		orm := &models.ORModels{}
+		if err := json.NewDecoder(resp.Body).Decode(orm); err != nil {
+			logger.Warn("failed to decode OR models for vision check", "error", err)
+			return false
+		}
+		return orm.HasVision(modelID)
+	default:
+		models, err := fetchLCPModelsWithStatus()
+		if err != nil {
+			logger.Warn("failed to fetch LCP models for vision check", "error", err)
+			return false
+		}
+		return models.HasVision(modelID)
+	}
+}
+
 // monitorModelLoad starts a goroutine that periodically checks if the specified model is loaded.
 func monitorModelLoad(modelID string) {
 	go func() {
@@ -569,7 +580,6 @@ func sendMsgToLLM(body io.Reader) {
 		streamDone <- true
 		return
 	}
-
 	// Check if the initial response is an error before starting to stream
 	if resp.StatusCode >= 400 {
 		// Read the response body to get detailed error information
@@ -584,7 +594,6 @@ func sendMsgToLLM(body io.Reader) {
 			streamDone <- true
 			return
 		}
-
 		// Parse the error response for detailed information
 		detailedError := extractDetailedErrorFromBytes(bodyBytes, resp.StatusCode)
 		logger.Error("API returned error status", "status_code", resp.StatusCode, "detailed_error", detailedError)
@@ -710,7 +719,6 @@ func sendMsgToLLM(body io.Reader) {
 				tokenCount++
 			}
 		}
-
 		// When we get content and have been streaming reasoning, close the thinking block
 		if chunk.Chunk != "" && hasReasoning && !reasoningSent {
 			// Close the thinking block before sending actual content
@@ -718,7 +726,6 @@ func sendMsgToLLM(body io.Reader) {
 			tokenCount++
 			reasoningSent = true
 		}
-
 		// bot sends way too many \n
 		answerText = strings.ReplaceAll(chunk.Chunk, "\n\n", "\n")
 		// Accumulate text to check for stop strings that might span across chunks
@@ -742,7 +749,7 @@ func sendMsgToLLM(body io.Reader) {
 		}
 	interrupt:
 		if interruptResp { // read bytes, so it would not get into beginning of the next req
-			interruptResp = false
+			// interruptResp = false
 			logger.Info("interrupted bot response", "chunk_counter", counter)
 			streamDone <- true
 			break
@@ -750,68 +757,6 @@ func sendMsgToLLM(body io.Reader) {
 	}
 }
 
-func chatRagUse(qText string) (string, error) {
-	logger.Debug("Starting RAG query", "original_query", qText)
-	tokenizer, err := english.NewSentenceTokenizer(nil)
-	if err != nil {
-		logger.Error("failed to create sentence tokenizer", "error", err)
-		return "", err
-	}
-	// this where llm should find the questions in text and ask them
-	questionsS := tokenizer.Tokenize(qText)
-	questions := make([]string, len(questionsS))
-	for i, q := range questionsS {
-		questions[i] = q.Text
-		logger.Debug("RAG question extracted", "index", i, "question", q.Text)
-	}
-
-	if len(questions) == 0 {
-		logger.Warn("No questions extracted from query text", "query", qText)
-		return "No related results from RAG vector storage.", nil
-	}
-
-	respVecs := []models.VectorRow{}
-	for i, q := range questions {
-		logger.Debug("Processing RAG question", "index", i, "question", q)
-		emb, err := ragger.LineToVector(q)
-		if err != nil {
-			logger.Error("failed to get embeddings for RAG", "error", err, "index", i, "question", q)
-			continue
-		}
-		logger.Debug("Got embeddings for question", "index", i, "question_len", len(q), "embedding_len", len(emb))
-
-		// Create EmbeddingResp struct for the search
-		embeddingResp := &models.EmbeddingResp{
-			Embedding: emb,
-			Index:     0, // Not used in search but required for the struct
-		}
-		vecs, err := ragger.SearchEmb(embeddingResp)
-		if err != nil {
-			logger.Error("failed to query embeddings in RAG", "error", err, "index", i, "question", q)
-			continue
-		}
-		logger.Debug("RAG search returned vectors", "index", i, "question", q, "vector_count", len(vecs))
-		respVecs = append(respVecs, vecs...)
-	}
-
-	// get raw text
-	resps := []string{}
-	logger.Debug("RAG query final results", "total_vecs_found", len(respVecs))
-	for _, rv := range respVecs {
-		resps = append(resps, rv.RawText)
-		logger.Debug("RAG result", "slug", rv.Slug, "filename", rv.FileName, "raw_text_len", len(rv.RawText))
-	}
-
-	if len(resps) == 0 {
-		logger.Info("No RAG results found for query", "original_query", qText, "question_count", len(questions))
-		return "No related results from RAG vector storage.", nil
-	}
-
-	result := strings.Join(resps, "\n")
-	logger.Debug("RAG query completed", "result_len", len(result), "response_count", len(resps))
-	return result, nil
-}
-
 func roleToIcon(role string) string {
 	return "<" + role + ">: "
 }
@@ -829,14 +774,47 @@ func chatWatcher(ctx context.Context) {
 	}
 }
 
+// inpired by https://github.com/rivo/tview/issues/225
+func showSpinner() {
+	spinners := []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"}
+	var i int
+	botPersona := cfg.AssistantRole
+	if cfg.WriteNextMsgAsCompletionAgent != "" {
+		botPersona = cfg.WriteNextMsgAsCompletionAgent
+	}
+	for botRespMode || toolRunningMode {
+		time.Sleep(400 * time.Millisecond)
+		spin := i % len(spinners)
+		app.QueueUpdateDraw(func() {
+			switch {
+			case toolRunningMode:
+				textArea.SetTitle(spinners[spin] + " tool")
+			case botRespMode:
+				textArea.SetTitle(spinners[spin] + " " + botPersona + " (F6 to interrupt)")
+			default:
+				textArea.SetTitle(spinners[spin] + " input")
+			}
+		})
+		i++
+	}
+	app.QueueUpdateDraw(func() {
+		textArea.SetTitle("input")
+	})
+}
+
 func chatRound(r *models.ChatRoundReq) error {
+	interruptResp = false
 	botRespMode = true
+	go showSpinner()
 	updateStatusLine()
 	botPersona := cfg.AssistantRole
 	if cfg.WriteNextMsgAsCompletionAgent != "" {
 		botPersona = cfg.WriteNextMsgAsCompletionAgent
 	}
-	defer func() { botRespMode = false }()
+	defer func() {
+		botRespMode = false
+		ClearImageAttachment()
+	}()
 	// check that there is a model set to use if is not local
 	choseChunkParser()
 	reader, err := chunkParser.FormMsg(r.UserMsg, r.Role, r.Resume)
@@ -855,13 +833,14 @@ func chatRound(r *models.ChatRoundReq) error {
 		chatBody.Messages = append(chatBody.Messages, models.RoleMsg{
 			Role: botPersona, Content: "",
 		})
-		fmt.Fprintf(textView, "\n[-:-:b](%d) ", msgIdx)
-		fmt.Fprint(textView, roleToIcon(botPersona))
-		fmt.Fprint(textView, "[-:-:-]\n")
-		if cfg.ThinkUse && !strings.Contains(cfg.CurrentAPI, "v1") {
-			// fmt.Fprint(textView, "<think>")
-			chunkChan <- "<think>"
+		nl := "\n\n"
+		prevText := textView.GetText(true)
+		if strings.HasSuffix(prevText, nl) {
+			nl = ""
+		} else if strings.HasSuffix(prevText, "\n") {
+			nl = "\n"
 		}
+		fmt.Fprintf(textView, "%s[-:-:b](%d) %s[-:-:-]\n", nl, msgIdx, roleToIcon(botPersona))
 	} else {
 		msgIdx = len(chatBody.Messages) - 1
 	}
@@ -988,7 +967,12 @@ out:
 	if err := updateStorageChat(activeChatName, chatBody.Messages); err != nil {
 		logger.Warn("failed to update storage", "error", err, "name", activeChatName)
 	}
-	if findCall(respText.String(), toolResp.String()) {
+	// Strip think blocks before parsing for tool calls
+	respTextNoThink := thinkBlockRE.ReplaceAllString(respText.String(), "")
+	if interruptResp {
+		return nil
+	}
+	if findCall(respTextNoThink, toolResp.String()) {
 		return nil
 	}
 	// Check if this message was sent privately to specific characters
@@ -1010,7 +994,7 @@ func cleanChatBody() {
 	}
 	// Tool request cleaning is now configurable via AutoCleanToolCallsFromCtx (default false)
 	// /completion msg where part meant for user and other part tool call
-	chatBody.Messages = cleanToolCalls(chatBody.Messages)
+	// chatBody.Messages = cleanToolCalls(chatBody.Messages)
 	chatBody.Messages = consolidateAssistantMessages(chatBody.Messages)
 }
 
@@ -1124,22 +1108,38 @@ func findCall(msg, toolCall string) bool {
 		}
 		lastToolCall.Args = openAIToolMap
 		fc = lastToolCall
-		// Set lastToolCall.ID from parsed tool call ID if available
-		if len(openAIToolMap) > 0 {
-			if id, exists := openAIToolMap["id"]; exists {
-				lastToolCall.ID = id
-			}
-		}
+		// NOTE: We do NOT override lastToolCall.ID from arguments.
+		// The ID should come from the streaming response (chunk.ToolID) set earlier.
+		// Some tools like todo_create have "id" in their arguments which is NOT the tool call ID.
 	} else {
 		jsStr := toolCallRE.FindString(msg)
 		if jsStr == "" { // no tool call case
 			return false
 		}
-		prefix := "__tool_call__\n"
-		suffix := "\n__tool_call__"
-		jsStr = strings.TrimSuffix(strings.TrimPrefix(jsStr, prefix), suffix)
+		// Remove prefix/suffix with flexible whitespace handling
+		jsStr = strings.TrimSpace(jsStr)
+		jsStr = strings.TrimPrefix(jsStr, "__tool_call__")
+		jsStr = strings.TrimSuffix(jsStr, "__tool_call__")
+		jsStr = strings.TrimSpace(jsStr)
 		// HTML-decode the JSON string to handle encoded characters like &lt; -> <=
 		decodedJsStr := html.UnescapeString(jsStr)
+		// Try to find valid JSON bounds (first { to last })
+		start := strings.Index(decodedJsStr, "{")
+		end := strings.LastIndex(decodedJsStr, "}")
+		if start == -1 || end == -1 || end <= start {
+			logger.Error("failed to find valid JSON in tool call", "json_string", decodedJsStr)
+			toolResponseMsg := models.RoleMsg{
+				Role:    cfg.ToolRole,
+				Content: "Error processing tool call: no valid JSON found. Please check the JSON format.",
+			}
+			chatBody.Messages = append(chatBody.Messages, toolResponseMsg)
+			crr := &models.ChatRoundReq{
+				Role: cfg.AssistantRole,
+			}
+			chatRoundChan <- crr
+			return true
+		}
+		decodedJsStr = decodedJsStr[start : end+1]
 		var err error
 		fc, err = unmarshalFuncCall(decodedJsStr)
 		if err != nil {
@@ -1166,14 +1166,18 @@ func findCall(msg, toolCall string) bool {
 		lastToolCall.Args = fc.Args
 	}
 	// we got here => last msg recognized as a tool call (correct or not)
-	// make sure it has ToolCallID
-	if chatBody.Messages[len(chatBody.Messages)-1].ToolCallID == "" {
-		// Tool call IDs should be alphanumeric strings with length 9!
-		chatBody.Messages[len(chatBody.Messages)-1].ToolCallID = randString(9)
+	// Use the tool call ID from streaming response (lastToolCall.ID)
+	// Don't generate random ID - the ID should match between assistant message and tool response
+	lastMsgIdx := len(chatBody.Messages) - 1
+	if lastToolCall.ID != "" {
+		chatBody.Messages[lastMsgIdx].ToolCallID = lastToolCall.ID
 	}
-	// Ensure lastToolCall.ID is set, fallback to assistant message's ToolCallID
-	if lastToolCall.ID == "" {
-		lastToolCall.ID = chatBody.Messages[len(chatBody.Messages)-1].ToolCallID
+	// Store tool call info in the assistant message
+	// Convert Args map to JSON string for storage
+	chatBody.Messages[lastMsgIdx].ToolCall = &models.ToolCall{
+		ID:   lastToolCall.ID,
+		Name: lastToolCall.Name,
+		Args: mapToString(lastToolCall.Args),
 	}
 	// call a func
 	_, ok := fnMap[fc.Name]
@@ -1198,17 +1202,66 @@ func findCall(msg, toolCall string) bool {
 		chatRoundChan <- crr
 		return true
 	}
+	// Show tool call progress indicator before execution
+	fmt.Fprintf(textView, "\n[yellow::i][tool: %s...][-:-:-]", fc.Name)
+	toolRunningMode = true
 	resp := callToolWithAgent(fc.Name, fc.Args)
-	toolMsg := string(resp) // Remove the "tool response: " prefix and %+v formatting
+	toolRunningMode = false
+	toolMsg := string(resp)
 	logger.Info("llm used a tool call", "tool_name", fc.Name, "too_args", fc.Args, "id", fc.ID, "tool_resp", toolMsg)
-	fmt.Fprintf(textView, "%s[-:-:b](%d) <%s>: [-:-:-]\n%s\n",
-		"\n\n", len(chatBody.Messages), cfg.ToolRole, toolMsg)
 	// Create tool response message with the proper tool_call_id
-	toolResponseMsg := models.RoleMsg{
-		Role:       cfg.ToolRole,
-		Content:    toolMsg,
-		ToolCallID: lastToolCall.ID, // Use the stored tool call ID
+	// Mark shell commands as always visible
+	isShellCommand := fc.Name == "execute_command"
+	// Check if response is multimodal content (image)
+	var toolResponseMsg models.RoleMsg
+	if strings.HasPrefix(strings.TrimSpace(toolMsg), `{"type":"multimodal_content"`) {
+		// Parse multimodal content response
+		multimodalResp := models.MultimodalToolResp{}
+		if err := json.Unmarshal([]byte(toolMsg), &multimodalResp); err == nil && multimodalResp.Type == "multimodal_content" {
+			// Create RoleMsg with ContentParts
+			var contentParts []any
+			for _, part := range multimodalResp.Parts {
+				partType := part["type"]
+				switch partType {
+				case "text":
+					contentParts = append(contentParts, models.TextContentPart{Type: "text", Text: part["text"]})
+				case "image_url":
+					contentParts = append(contentParts, models.ImageContentPart{
+						Type: "image_url",
+						ImageURL: struct {
+							URL string `json:"url"`
+						}{URL: part["url"]},
+					})
+				default:
+					continue
+				}
+			}
+			toolResponseMsg = models.RoleMsg{
+				Role:            cfg.ToolRole,
+				ContentParts:    contentParts,
+				HasContentParts: true,
+				ToolCallID:      lastToolCall.ID,
+				IsShellCommand:  isShellCommand,
+			}
+		} else {
+			// Fallback to regular content
+			toolResponseMsg = models.RoleMsg{
+				Role:           cfg.ToolRole,
+				Content:        toolMsg,
+				ToolCallID:     lastToolCall.ID,
+				IsShellCommand: isShellCommand,
+			}
+		}
+	} else {
+		toolResponseMsg = models.RoleMsg{
+			Role:           cfg.ToolRole,
+			Content:        toolMsg,
+			ToolCallID:     lastToolCall.ID,
+			IsShellCommand: isShellCommand,
+		}
 	}
+	fmt.Fprintf(textView, "%s[-:-:b](%d) <%s>: [-:-:-]\n%s\n",
+		"\n\n", len(chatBody.Messages), cfg.ToolRole, toolResponseMsg.GetText())
 	chatBody.Messages = append(chatBody.Messages, toolResponseMsg)
 	logger.Debug("findCall: added actual tool response", "role", toolResponseMsg.Role, "content_len", len(toolResponseMsg.Content), "tool_call_id", toolResponseMsg.ToolCallID, "message_count_after_add", len(chatBody.Messages))
 	// Clear the stored tool call ID after using it
@@ -1224,12 +1277,42 @@ func findCall(msg, toolCall string) bool {
 
 func chatToTextSlice(messages []models.RoleMsg, showSys bool) []string {
 	resp := make([]string, len(messages))
-	for i, msg := range messages {
-		// INFO: skips system msg and tool msg
-		if !showSys && (msg.Role == cfg.ToolRole || msg.Role == "system") {
+	for i := range messages {
+		icon := fmt.Sprintf("[-:-:b](%d) <%s>:[-:-:-]", i, messages[i].Role)
+		// Handle tool call indicators (assistant messages with tool call but empty content)
+		if messages[i].Role == cfg.AssistantRole && messages[i].ToolCall != nil && messages[i].ToolCall.ID != "" {
+			// This is a tool call indicator - show collapsed
+			if toolCollapsed {
+				toolName := messages[i].ToolCall.Name
+				resp[i] = strings.ReplaceAll(fmt.Sprintf("%s\n%s\n[yellow::i][tool call: %s (press Ctrl+T to expand)][-:-:-]\n", icon, messages[i].GetText(), toolName), "\n\n", "\n")
+			} else {
+				// Show full tool call info
+				toolName := messages[i].ToolCall.Name
+				resp[i] = strings.ReplaceAll(fmt.Sprintf("%s\n%s\n[yellow::i][tool call: %s][-:-:-]\nargs: %s\nid: %s\n", icon, messages[i].GetText(), toolName, messages[i].ToolCall.Args, messages[i].ToolCall.ID), "\n\n", "\n")
+			}
 			continue
 		}
-		resp[i] = msg.ToText(i)
+		// Handle tool responses
+		if messages[i].Role == cfg.ToolRole || messages[i].Role == "tool" {
+			// Always show shell commands
+			if messages[i].IsShellCommand {
+				resp[i] = MsgToText(i, &messages[i])
+				continue
+			}
+			// Hide non-shell tool responses when collapsed
+			if toolCollapsed {
+				resp[i] = icon + "\n[yellow::i][tool resp (press Ctrl+T to expand)][-:-:-]\n"
+				continue
+			}
+			// When expanded, show tool responses
+			resp[i] = MsgToText(i, &messages[i])
+			continue
+		}
+		// INFO: skips system msg when showSys is false
+		if !showSys && messages[i].Role == "system" {
+			continue
+		}
+		resp[i] = MsgToText(i, &messages[i])
 	}
 	return resp
 }
@@ -1237,7 +1320,6 @@ func chatToTextSlice(messages []models.RoleMsg, showSys bool) []string {
 func chatToText(messages []models.RoleMsg, showSys bool) string {
 	s := chatToTextSlice(messages, showSys)
 	text := strings.Join(s, "\n")
-
 	// Collapse thinking blocks if enabled
 	if thinkingCollapsed {
 		text = thinkRE.ReplaceAllStringFunc(text, func(match string) string {
@@ -1261,27 +1343,9 @@ func chatToText(messages []models.RoleMsg, showSys bool) string {
 			}
 		}
 	}
-
 	return text
 }
 
-func removeThinking(chatBody *models.ChatBody) {
-	msgs := []models.RoleMsg{}
-	for _, msg := range chatBody.Messages {
-		// Filter out tool messages and thinking markers
-		if msg.Role == cfg.ToolRole {
-			continue
-		}
-		// find thinking and remove it
-		rm := models.RoleMsg{
-			Role:    msg.Role,
-			Content: thinkRE.ReplaceAllString(msg.Content, ""),
-		}
-		msgs = append(msgs, rm)
-	}
-	chatBody.Messages = msgs
-}
-
 func addNewChat(chatName string) {
 	id, err := store.ChatGetMaxID()
 	if err != nil {
@@ -1336,11 +1400,28 @@ func updateModelLists() {
 	}
 	// if llama.cpp started after gf-lt?
 	localModelsMu.Lock()
-	LocalModels, err = fetchLCPModels()
+	LocalModels, err = fetchLCPModelsWithLoadStatus()
 	localModelsMu.Unlock()
 	if err != nil {
 		logger.Warn("failed to fetch llama.cpp models", "error", err)
 	}
+	// set already loaded model in llama.cpp
+	if strings.Contains(cfg.CurrentAPI, "localhost") || strings.Contains(cfg.CurrentAPI, "127.0.0.1") {
+		localModelsMu.Lock()
+		defer localModelsMu.Unlock()
+		for i := range LocalModels {
+			if strings.Contains(LocalModels[i], models.LoadedMark) {
+				m := strings.TrimPrefix(LocalModels[i], models.LoadedMark)
+				cfg.CurrentModel = m
+				chatBody.Model = m
+				cachedModelColor = "green"
+				updateStatusLine()
+				UpdateToolCapabilities()
+				app.Draw()
+				return
+			}
+		}
+	}
 }
 
 func refreshLocalModelsIfEmpty() {
@@ -1403,15 +1484,6 @@ func init() {
 		os.Exit(1)
 		return
 	}
-	// Set image base directory for path display
-	baseDir := cfg.FilePickerDir
-	if baseDir == "" || baseDir == "." {
-		// Resolve "." to current working directory
-		if wd, err := os.Getwd(); err == nil {
-			baseDir = wd
-		}
-	}
-	models.SetImageBaseDir(baseDir)
 	defaultStarter = []models.RoleMsg{
 		{Role: "system", Content: basicSysMsg},
 		{Role: cfg.AssistantRole, Content: defaultFirstMsg},
@@ -1426,8 +1498,6 @@ func init() {
 	}
 	// load cards
 	basicCard.Role = cfg.AssistantRole
-	// toolCard.Role = cfg.AssistantRole
-	//
 	logLevel.Set(slog.LevelInfo)
 	logger = slog.New(slog.NewTextHandler(logfile, &slog.HandlerOptions{Level: logLevel}))
 	store = storage.NewProviderSQL(cfg.DBPATH, logger)

bot_test.go

@@ -1,12 +1,10 @@
 package main
-
 import (
 	"gf-lt/config"
 	"gf-lt/models"
 	"reflect"
 	"testing"
 )
-
 func TestConsolidateConsecutiveAssistantMessages(t *testing.T) {
 	// Mock config for testing
 	testCfg := &config.Config{
@@ -14,7 +12,6 @@ func TestConsolidateConsecutiveAssistantMessages(t *testing.T) {
 		WriteNextMsgAsCompletionAgent: "",
 	}
 	cfg = testCfg
-
 	tests := []struct {
 		name     string
 		input    []models.RoleMsg
@@ -114,38 +111,31 @@ func TestConsolidateConsecutiveAssistantMessages(t *testing.T) {
 			},
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			result := consolidateAssistantMessages(tt.input)
-
 			if len(result) != len(tt.expected) {
 				t.Errorf("Expected %d messages, got %d", len(tt.expected), len(result))
 				t.Logf("Result: %+v", result)
 				t.Logf("Expected: %+v", tt.expected)
 				return
 			}
-
 			for i, expectedMsg := range tt.expected {
 				if i >= len(result) {
 					t.Errorf("Result has fewer messages than expected at index %d", i)
 					continue
 				}
-
 				actualMsg := result[i]
 				if actualMsg.Role != expectedMsg.Role {
 					t.Errorf("Message %d: expected role '%s', got '%s'", i, expectedMsg.Role, actualMsg.Role)
 				}
-
 				if actualMsg.Content != expectedMsg.Content {
 					t.Errorf("Message %d: expected content '%s', got '%s'", i, expectedMsg.Content, actualMsg.Content)
 				}
-
 				if actualMsg.ToolCallID != expectedMsg.ToolCallID {
 					t.Errorf("Message %d: expected ToolCallID '%s', got '%s'", i, expectedMsg.ToolCallID, actualMsg.ToolCallID)
 				}
 			}
-
 			// Additional check: ensure no messages were lost
 			if !reflect.DeepEqual(result, tt.expected) {
 				t.Errorf("Result does not match expected:\nResult:   %+v\nExpected: %+v", result, tt.expected)
@@ -153,7 +143,6 @@ func TestConsolidateConsecutiveAssistantMessages(t *testing.T) {
 		})
 	}
 }
-
 func TestUnmarshalFuncCall(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -213,7 +202,6 @@ func TestUnmarshalFuncCall(t *testing.T) {
 			wantErr: true,
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := unmarshalFuncCall(tt.jsonStr)
@@ -238,7 +226,6 @@ func TestUnmarshalFuncCall(t *testing.T) {
 		})
 	}
 }
-
 func TestConvertJSONToMapStringString(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -265,7 +252,6 @@ func TestConvertJSONToMapStringString(t *testing.T) {
 			wantErr: true,
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := convertJSONToMapStringString(tt.jsonStr)
@@ -287,7 +273,6 @@ func TestConvertJSONToMapStringString(t *testing.T) {
 		})
 	}
 }
-
 func TestParseKnownToTag(t *testing.T) {
 	tests := []struct {
 		name        string
@@ -378,7 +363,6 @@ func TestParseKnownToTag(t *testing.T) {
 			wantKnownTo: []string{"Alice", "Bob", "Carl"},
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			// Set up config
@@ -402,7 +386,6 @@ func TestParseKnownToTag(t *testing.T) {
 		})
 	}
 }
-
 func TestProcessMessageTag(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -498,7 +481,6 @@ func TestProcessMessageTag(t *testing.T) {
 			},
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			testCfg := &config.Config{
@@ -529,7 +511,6 @@ func TestProcessMessageTag(t *testing.T) {
 		})
 	}
 }
-
 func TestFilterMessagesForCharacter(t *testing.T) {
 	messages := []models.RoleMsg{
 		{Role: "system", Content: "System message", KnownTo: nil}, // visible to all
@@ -539,7 +520,6 @@ func TestFilterMessagesForCharacter(t *testing.T) {
 		{Role: "Alice", Content: "Private to Carl", KnownTo: []string{"Alice", "Carl"}},
 		{Role: "Carl", Content: "Hi all", KnownTo: nil}, // visible to all
 	}
-
 	tests := []struct {
 		name        string
 		enabled     bool
@@ -583,7 +563,6 @@ func TestFilterMessagesForCharacter(t *testing.T) {
 			wantIndices: []int{0, 1, 5},
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			testCfg := &config.Config{
@@ -591,15 +570,12 @@ func TestFilterMessagesForCharacter(t *testing.T) {
 				CharSpecificContextTag:     "@",
 			}
 			cfg = testCfg
-
 			got := filterMessagesForCharacter(messages, tt.character)
-
 			if len(got) != len(tt.wantIndices) {
 				t.Errorf("filterMessagesForCharacter() returned %d messages, want %d", len(got), len(tt.wantIndices))
 				t.Logf("got: %v", got)
 				return
 			}
-
 			for i, idx := range tt.wantIndices {
 				if got[i].Content != messages[idx].Content {
 					t.Errorf("filterMessagesForCharacter() message %d content = %q, want %q", i, got[i].Content, messages[idx].Content)
@@ -608,7 +584,6 @@ func TestFilterMessagesForCharacter(t *testing.T) {
 		})
 	}
 }
-
 func TestRoleMsgCopyPreservesKnownTo(t *testing.T) {
 	// Test that the Copy() method preserves the KnownTo field
 	originalMsg := models.RoleMsg{
@@ -616,9 +591,7 @@ func TestRoleMsgCopyPreservesKnownTo(t *testing.T) {
 		Content: "Test message",
 		KnownTo: []string{"Bob", "Charlie"},
 	}
-
 	copiedMsg := originalMsg.Copy()
-
 	if copiedMsg.Role != originalMsg.Role {
 		t.Errorf("Copy() failed to preserve Role: got %q, want %q", copiedMsg.Role, originalMsg.Role)
 	}
@@ -635,7 +608,6 @@ func TestRoleMsgCopyPreservesKnownTo(t *testing.T) {
 		t.Errorf("Copy() failed to preserve hasContentParts flag")
 	}
 }
-
 func TestKnownToFieldPreservationScenario(t *testing.T) {
 	// Test the specific scenario from the log where KnownTo field was getting lost
 	originalMsg := models.RoleMsg{
@@ -643,28 +615,22 @@ func TestKnownToFieldPreservationScenario(t *testing.T) {
 		Content: `Alice: "Okay, Bob. The word is... **'Ephemeral'**. (ooc: @Bob@)"`,
 		KnownTo: []string{"Bob"}, // This was detected in the log
 	}
-
 	t.Logf("Original message - Role: %s, Content: %s, KnownTo: %v",
 		originalMsg.Role, originalMsg.Content, originalMsg.KnownTo)
-
 	// Simulate what happens when the message gets copied during processing
 	copiedMsg := originalMsg.Copy()
-
 	t.Logf("Copied message - Role: %s, Content: %s, KnownTo: %v",
 		copiedMsg.Role, copiedMsg.Content, copiedMsg.KnownTo)
-
 	// Check if KnownTo field survived the copy
 	if len(copiedMsg.KnownTo) == 0 {
 		t.Error("ERROR: KnownTo field was lost during copy!")
 	} else {
 		t.Log("SUCCESS: KnownTo field was preserved during copy!")
 	}
-
 	// Verify the content is the same
 	if copiedMsg.Content != originalMsg.Content {
 		t.Errorf("Content was changed during copy: got %s, want %s", copiedMsg.Content, originalMsg.Content)
 	}
-
 	// Verify the KnownTo slice is properly copied
 	if !reflect.DeepEqual(copiedMsg.KnownTo, originalMsg.KnownTo) {
 		t.Errorf("KnownTo was not properly copied: got %v, want %v", copiedMsg.KnownTo, originalMsg.KnownTo)

config.example.toml

@@ -12,7 +12,7 @@ OpenRouterChatAPI = "https://openrouter.ai/api/v1/chat/completions"
 # OpenRouterToken = ""
 # embeddings
 EmbedURL = "http://localhost:8082/v1/embeddings"
-HFToken = false
+HFToken = ""
 #
 ShowSys = true
 LogFile = "log.txt"
@@ -27,7 +27,6 @@ AutoCleanToolCallsFromCtx = false
 RAGEnabled = false
 RAGBatchSize = 1
 RAGWordLimit = 80
-RAGWorkers = 2
 RAGDir = "ragimport"
 # extra tts
 TTS_ENABLED = false

config/config.go

@@ -18,7 +18,6 @@ type Config struct {
 	UserRole                      string `toml:"UserRole"`
 	ToolRole                      string `toml:"ToolRole"`
 	ToolUse                       bool   `toml:"ToolUse"`
-	ThinkUse                      bool   `toml:"ThinkUse"`
 	StripThinkingFromAPI          bool   `toml:"StripThinkingFromAPI"`
 	ReasoningEffort               string `toml:"ReasoningEffort"`
 	AssistantRole                 string `toml:"AssistantRole"`
@@ -40,7 +39,6 @@ type Config struct {
 	// rag settings
 	RAGEnabled   bool   `toml:"RAGEnabled"`
 	RAGDir       string `toml:"RAGDir"`
-	RAGWorkers   uint32 `toml:"RAGWorkers"`
 	RAGBatchSize int    `toml:"RAGBatchSize"`
 	RAGWordLimit uint32 `toml:"RAGWordLimit"`
 	// deepseek
@@ -125,6 +123,9 @@ func LoadConfig(fn string) (*Config, error) {
 	if config.CompletionAPI != "" {
 		config.ApiLinks = append(config.ApiLinks, config.CompletionAPI)
 	}
+	if config.RAGDir == "" {
+		config.RAGDir = "ragimport"
+	}
 	// if any value is empty fill with default
 	return config, nil
 }

docs/config.md

@@ -80,9 +80,6 @@ This document explains how to set up and configure the application using the `co
 #### RAGWordLimit (`80`)
 - Maximum number of words in a batch to tokenize and store.
 
-#### RAGWorkers (`2`)
-- Number of concurrent workers for RAG processing.
-
 #### RAGDir (`"ragimport"`)
 - Directory containing documents for RAG processing.
 
@@ -165,9 +162,6 @@ Those could be switched in program, but also bould be setup in config.
 #### ToolUse
 - Enable or disable explanation of tools to llm, so it could use them.
 
-#### ThinkUse
-- Enable or disable insertion of  JsonSerializerToken at the beggining of llm resp.
-
 ### StripThinkingFromAPI (`true`)
 - Strip thinking blocks from messages before sending to LLM. Keeps them in chat history for local viewing but reduces token usage in API calls.
 

go.mod

@@ -6,17 +6,19 @@ require (
 	github.com/BurntSushi/toml v1.5.0
 	github.com/GrailFinder/google-translate-tts v0.1.3
 	github.com/GrailFinder/searchagent v0.2.0
+	github.com/PuerkitoBio/goquery v1.11.0
 	github.com/gdamore/tcell/v2 v2.13.2
 	github.com/glebarez/go-sqlite v1.22.0
 	github.com/gopxl/beep/v2 v2.1.1
 	github.com/gordonklaus/portaudio v0.0.0-20250206071425-98a94950218b
 	github.com/jmoiron/sqlx v1.4.0
+	github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728
 	github.com/neurosnap/sentences v1.1.2
 	github.com/rivo/tview v0.42.0
+	github.com/yuin/goldmark v1.4.13
 )
 
 require (
-	github.com/PuerkitoBio/goquery v1.11.0 // indirect
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/ebitengine/oto/v3 v3.4.0 // indirect

go.sum

@@ -43,6 +43,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
 github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
+github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+ExRDqGQltzXqN/xypdKP86niVn8=
+github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
@@ -67,6 +69,7 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=

helpfuncs.go

@@ -11,12 +11,11 @@ import (
 	"path"
 	"path/filepath"
 	"slices"
+	"strconv"
 	"strings"
 	"time"
 	"unicode"
 
-	"math/rand/v2"
-
 	"github.com/rivo/tview"
 )
 
@@ -29,10 +28,8 @@ func startModelColorUpdater() {
 	go func() {
 		ticker := time.NewTicker(5 * time.Second)
 		defer ticker.Stop()
-
 		// Initial check
 		updateCachedModelColor()
-
 		for range ticker.C {
 			updateCachedModelColor()
 		}
@@ -45,7 +42,6 @@ func updateCachedModelColor() {
 		cachedModelColor = "orange"
 		return
 	}
-
 	// Check if model is loaded
 	loaded, err := isModelLoaded(chatBody.Model)
 	if err != nil {
@@ -69,21 +65,30 @@ func isASCII(s string) bool {
 	return true
 }
 
+func mapToString[V any](m map[string]V) string {
+	rs := strings.Builder{}
+	for k, v := range m {
+		fmt.Fprintf(&rs, "%v: %v\n", k, v)
+	}
+	return rs.String()
+}
+
 // stripThinkingFromMsg removes thinking blocks from assistant messages.
 // Skips user, tool, and system messages as they may contain thinking examples.
 func stripThinkingFromMsg(msg *models.RoleMsg) *models.RoleMsg {
 	if !cfg.StripThinkingFromAPI {
 		return msg
 	}
-	// Skip user, tool, and system messages - they might contain thinking examples
+	// Skip user, tool, they might contain thinking and system messages - examples
 	if msg.Role == cfg.UserRole || msg.Role == cfg.ToolRole || msg.Role == "system" {
 		return msg
 	}
 	// Strip thinking from assistant messages
-	if thinkRE.MatchString(msg.Content) {
-		msg.Content = thinkRE.ReplaceAllString(msg.Content, "")
-		// Clean up any double newlines that might result
-		msg.Content = strings.TrimSpace(msg.Content)
+	msgText := msg.GetText()
+	if thinkRE.MatchString(msgText) {
+		cleanedText := thinkRE.ReplaceAllString(msgText, "")
+		cleanedText = strings.TrimSpace(cleanedText)
+		msg.SetText(cleanedText)
 	}
 	return msg
 }
@@ -211,8 +216,10 @@ func startNewChat(keepSysP bool) {
 	chatBody.Messages = chatBody.Messages[:2]
 	textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
 	newChat := &models.Chat{
-		ID:   id + 1,
-		Name: fmt.Sprintf("%d_%s", id+1, cfg.AssistantRole),
+		ID:        id + 1,
+		Name:      fmt.Sprintf("%d_%s", id+1, cfg.AssistantRole),
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
 		// chat is written to db when we get first llm response (or any)
 		// actual chat history (messages) would be parsed then
 		Msgs:  "",
@@ -357,7 +364,7 @@ func makeStatusLine() string {
 	}
 	// Get model color based on load status for local llama.cpp models
 	modelColor := getModelColor()
-	statusLine := fmt.Sprintf(statusLineTempl, boolColors[botRespMode], activeChatName,
+	statusLine := fmt.Sprintf(statusLineTempl, activeChatName,
 		boolColors[cfg.ToolUse], modelColor, chatBody.Model, boolColors[cfg.SkipLLMResp],
 		cfg.CurrentAPI, persona, botPersona)
 	if cfg.STT_ENABLED {
@@ -370,17 +377,88 @@ func makeStatusLine() string {
 		roleInject := fmt.Sprintf(" | [%s:-:b]role injection[-:-:-] (alt+7)", boolColors[injectRole])
 		statusLine += roleInject
 	}
+	// context tokens
+	contextTokens := getContextTokens()
+	maxCtx := getMaxContextTokens()
+	if maxCtx == 0 {
+		maxCtx = 16384
+	}
+	if contextTokens > 0 {
+		contextInfo := fmt.Sprintf(" | context-estim: [orange:-:b]%d/%d[-:-:-]", contextTokens, maxCtx)
+		statusLine += contextInfo
+	}
 	return statusLine + imageInfo + shellModeInfo
 }
 
-var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
-
-func randString(n int) string {
-	b := make([]rune, n)
-	for i := range b {
-		b[i] = letters[rand.IntN(len(letters))]
+func getContextTokens() int {
+	if chatBody == nil || chatBody.Messages == nil {
+		return 0
 	}
-	return string(b)
+	total := 0
+	messages := chatBody.Messages
+	for i := range messages {
+		msg := &messages[i]
+		if msg.Stats != nil && msg.Stats.Tokens > 0 {
+			total += msg.Stats.Tokens
+		} else if msg.GetText() != "" {
+			total += len(msg.GetText()) / 4
+		}
+	}
+	return total
+}
+
+const deepseekContext = 128000
+
+func getMaxContextTokens() int {
+	if chatBody == nil || chatBody.Model == "" {
+		return 0
+	}
+	modelName := chatBody.Model
+	switch {
+	case strings.Contains(cfg.CurrentAPI, "openrouter"):
+		if orModelsData != nil {
+			for i := range orModelsData.Data {
+				m := &orModelsData.Data[i]
+				if m.ID == modelName {
+					return m.ContextLength
+				}
+			}
+		}
+	case strings.Contains(cfg.CurrentAPI, "deepseek"):
+		return deepseekContext
+	default:
+		if localModelsData != nil {
+			for i := range localModelsData.Data {
+				m := &localModelsData.Data[i]
+				if m.ID == modelName {
+					for _, arg := range m.Status.Args {
+						if strings.HasPrefix(arg, "--ctx-size") {
+							if strings.Contains(arg, "=") {
+								val := strings.Split(arg, "=")[1]
+								if n, err := strconv.Atoi(val); err == nil {
+									return n
+								}
+							} else {
+								idx := -1
+								for j, a := range m.Status.Args {
+									if a == "--ctx-size" && j+1 < len(m.Status.Args) {
+										idx = j + 1
+										break
+									}
+								}
+								if idx != -1 {
+									if n, err := strconv.Atoi(m.Status.Args[idx]); err == nil {
+										return n
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	return 0
 }
 
 // set of roles within card definition and mention in chat history
@@ -426,12 +504,11 @@ func deepseekModelValidator() error {
 
 func toggleShellMode() {
 	shellMode = !shellMode
+	setShellMode(shellMode)
 	if shellMode {
-		// Update input placeholder to indicate shell mode
-		textArea.SetPlaceholder("SHELL MODE: Enter command and press <Esc> to execute")
+		shellInput.SetLabel(fmt.Sprintf("[%s]$ ", cfg.FilePickerDir))
 	} else {
-		// Reset to normal mode
-		textArea.SetPlaceholder("input is multiline; press <Enter> to start the next line;\npress <Esc> to send the message. Alt+1 to exit shell mode")
+		textArea.SetPlaceholder("input is multiline; press <Enter> to start the next line;\npress <Esc> to send the message.")
 	}
 	updateStatusLine()
 }
@@ -443,23 +520,29 @@ func updateFlexLayout() {
 	}
 	flex.Clear()
 	flex.AddItem(textView, 0, 40, false)
-	flex.AddItem(textArea, 0, 10, false)
+	if shellMode {
+		flex.AddItem(shellInput, 0, 10, false)
+	} else {
+		flex.AddItem(textArea, 0, 10, false)
+	}
 	if positionVisible {
 		flex.AddItem(statusLineWidget, 0, 2, false)
 	}
 	// Keep focus on currently focused widget
 	focused := app.GetFocus()
-	if focused == textView {
+	switch {
+	case focused == textView:
 		app.SetFocus(textView)
-	} else {
+	case shellMode:
+		app.SetFocus(shellInput)
+	default:
 		app.SetFocus(textArea)
 	}
 }
 
 func executeCommandAndDisplay(cmdText string) {
-	// Parse the command (split by spaces, but handle quoted arguments)
-	cmdParts := parseCommand(cmdText)
-	if len(cmdParts) == 0 {
+	cmdText = strings.TrimSpace(cmdText)
+	if cmdText == "" {
 		fmt.Fprintf(textView, "\n[red]Error: No command provided[-:-:-]\n")
 		if scrollToEndEnabled {
 			textView.ScrollToEnd()
@@ -467,17 +550,63 @@ func executeCommandAndDisplay(cmdText string) {
 		colorText()
 		return
 	}
-	command := cmdParts[0]
-	args := []string{}
-	if len(cmdParts) > 1 {
-		args = cmdParts[1:]
+	workingDir := cfg.FilePickerDir
+	// Handle cd command specially to update working directory
+	if strings.HasPrefix(cmdText, "cd ") {
+		newDir := strings.TrimPrefix(cmdText, "cd ")
+		newDir = strings.TrimSpace(newDir)
+		// Handle cd ~ or cdHOME
+		if strings.HasPrefix(newDir, "~") {
+			home := os.Getenv("HOME")
+			newDir = strings.Replace(newDir, "~", home, 1)
+		}
+		// Check if directory exists
+		if _, err := os.Stat(newDir); err == nil {
+			workingDir = newDir
+			cfg.FilePickerDir = workingDir
+			// Update shell input label with new directory
+			shellInput.SetLabel(fmt.Sprintf("[%s]$ ", cfg.FilePickerDir))
+			outputContent := workingDir
+			// Add the command being executed to the chat
+			fmt.Fprintf(textView, "\n[-:-:b](%d) <%s>: [-:-:-]\n$ %s\n",
+				len(chatBody.Messages), cfg.ToolRole, cmdText)
+			fmt.Fprintf(textView, "%s\n", outputContent)
+			combinedMsg := models.RoleMsg{
+				Role:    cfg.ToolRole,
+				Content: "$ " + cmdText + "\n\n" + outputContent,
+			}
+			chatBody.Messages = append(chatBody.Messages, combinedMsg)
+			if scrollToEndEnabled {
+				textView.ScrollToEnd()
+			}
+			colorText()
+			return
+		} else {
+			outputContent := "cd: " + newDir + ": No such file or directory"
+			fmt.Fprintf(textView, "\n[-:-:b](%d) <%s>: [-:-:-]\n$ %s\n",
+				len(chatBody.Messages), cfg.ToolRole, cmdText)
+			fmt.Fprintf(textView, "[red]%s[-:-:-]\n", outputContent)
+			combinedMsg := models.RoleMsg{
+				Role:    cfg.ToolRole,
+				Content: "$ " + cmdText + "\n\n" + outputContent,
+			}
+			chatBody.Messages = append(chatBody.Messages, combinedMsg)
+			if scrollToEndEnabled {
+				textView.ScrollToEnd()
+			}
+			colorText()
+			return
+		}
 	}
-	// Create the command execution
-	cmd := exec.Command(command, args...)
+
+	// Use /bin/sh to support pipes, redirects, etc.
+	cmd := exec.Command("/bin/sh", "-c", cmdText)
+	cmd.Dir = workingDir
 	// Execute the command and get output
 	output, err := cmd.CombinedOutput()
 	// Add the command being executed to the chat
-	fmt.Fprintf(textView, "\n[yellow]$ %s[-:-:-]\n", cmdText)
+	fmt.Fprintf(textView, "\n[-:-:b](%d) <%s>: [-:-:-]\n$ %s\n",
+		len(chatBody.Messages), cfg.ToolRole, cmdText)
 	var outputContent string
 	if err != nil {
 		// Include both output and error
@@ -514,42 +643,11 @@ func executeCommandAndDisplay(cmdText string) {
 		textView.ScrollToEnd()
 	}
 	colorText()
-}
-
-// parseCommand splits command string handling quotes properly
-func parseCommand(cmd string) []string {
-	var args []string
-	var current string
-	var inQuotes bool
-	var quoteChar rune
-	for _, r := range cmd {
-		switch r {
-		case '"', '\'':
-			if inQuotes {
-				if r == quoteChar {
-					inQuotes = false
-				} else {
-					current += string(r)
-				}
-			} else {
-				inQuotes = true
-				quoteChar = r
-			}
-		case ' ', '\t':
-			if inQuotes {
-				current += string(r)
-			} else if current != "" {
-				args = append(args, current)
-				current = ""
-			}
-		default:
-			current += string(r)
-		}
+	// Add command to history (avoid duplicates at the end)
+	if len(shellHistory) == 0 || shellHistory[len(shellHistory)-1] != cmdText {
+		shellHistory = append(shellHistory, cmdText)
 	}
-	if current != "" {
-		args = append(args, current)
-	}
-	return args
+	shellHistoryPos = -1
 }
 
 // == search ==
@@ -791,3 +889,91 @@ func scanFiles(dir, filter string) []string {
 	scanRecursive(dir, 0, "")
 	return files
 }
+
+// models logic that is too complex for models package
+func MsgToText(i int, m *models.RoleMsg) string {
+	var contentStr string
+	var imageIndicators []string
+	if !m.HasContentParts {
+		contentStr = m.Content
+	} else {
+		var textParts []string
+		for _, part := range m.ContentParts {
+			switch p := part.(type) {
+			case models.TextContentPart:
+				if p.Type == "text" {
+					textParts = append(textParts, p.Text)
+				}
+			case models.ImageContentPart:
+				displayPath := p.Path
+				if displayPath == "" {
+					displayPath = "image"
+				} else {
+					displayPath = extractDisplayPath(displayPath, cfg.FilePickerDir)
+				}
+				imageIndicators = append(imageIndicators, fmt.Sprintf("[orange::i][image: %s][-:-:-]", displayPath))
+			case map[string]any:
+				if partType, exists := p["type"]; exists {
+					switch partType {
+					case "text":
+						if textVal, textExists := p["text"]; textExists {
+							if textStr, isStr := textVal.(string); isStr {
+								textParts = append(textParts, textStr)
+							}
+						}
+					case "image_url":
+						var displayPath string
+						if pathVal, pathExists := p["path"]; pathExists {
+							if pathStr, isStr := pathVal.(string); isStr && pathStr != "" {
+								displayPath = extractDisplayPath(pathStr, cfg.FilePickerDir)
+							}
+						}
+						if displayPath == "" {
+							displayPath = "image"
+						}
+						imageIndicators = append(imageIndicators, fmt.Sprintf("[orange::i][image: %s][-:-:-]", displayPath))
+					}
+				}
+			}
+		}
+		contentStr = strings.Join(textParts, " ") + " "
+	}
+	contentStr, _ = strings.CutPrefix(contentStr, m.Role+":")
+	icon := fmt.Sprintf("(%d) <%s>: ", i, m.Role)
+	var finalContent strings.Builder
+	if len(imageIndicators) > 0 {
+		for _, indicator := range imageIndicators {
+			finalContent.WriteString(indicator)
+			finalContent.WriteString("\n")
+		}
+	}
+	finalContent.WriteString(contentStr)
+	if m.Stats != nil {
+		fmt.Fprintf(&finalContent, "\n[gray::i][%d tok, %.1fs, %.1f t/s][-:-:-]", m.Stats.Tokens, m.Stats.Duration, m.Stats.TokensPerSec)
+	}
+	textMsg := fmt.Sprintf("[-:-:b]%s[-:-:-]\n%s\n", icon, finalContent.String())
+	return strings.ReplaceAll(textMsg, "\n\n", "\n")
+}
+
+// extractDisplayPath returns a path suitable for display, potentially relative to imageBaseDir
+func extractDisplayPath(p, bp string) string {
+	if p == "" {
+		return ""
+	}
+	// If base directory is set, try to make path relative to it
+	if bp != "" {
+		if rel, err := filepath.Rel(bp, p); err == nil {
+			// Check if relative path doesn't start with ".." (meaning it's within base dir)
+			// If it starts with "..", we might still want to show it as relative
+			// but for now we show full path if it goes outside base dir
+			if !strings.HasPrefix(rel, "..") {
+				p = rel
+			}
+		}
+	}
+	// Truncate long paths to last 60 characters if needed
+	if len(p) > 60 {
+		return "..." + p[len(p)-60:]
+	}
+	return p
+}

llm.go

@@ -3,7 +3,6 @@ package main
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"gf-lt/models"
 	"io"
 	"strings"
@@ -11,12 +10,11 @@ import (
 
 var imageAttachmentPath string // Global variable to track image attachment for next message
 var lastImg string             // for ctrl+j
-var RAGMsg = "Retrieved context for user's query:\n"
 
 // containsToolSysMsg checks if the toolSysMsg already exists in the chat body
 func containsToolSysMsg() bool {
-	for _, msg := range chatBody.Messages {
-		if msg.Role == cfg.ToolRole && msg.Content == toolSysMsg {
+	for i := range chatBody.Messages {
+		if chatBody.Messages[i].Role == cfg.ToolRole && chatBody.Messages[i].Content == toolSysMsg {
 			return true
 		}
 	}
@@ -120,73 +118,69 @@ func (lcp LCPCompletion) FormMsg(msg, role string, resume bool) (io.Reader, erro
 	logger.Debug("formmsg lcpcompletion", "link", cfg.CurrentAPI)
 	localImageAttachmentPath := imageAttachmentPath
 	var multimodalData []string
-	if localImageAttachmentPath != "" {
-		imageURL, err := models.CreateImageURLFromPath(localImageAttachmentPath)
-		if err != nil {
-			logger.Error("failed to create image URL from path for completion",
-				"error", err, "path", localImageAttachmentPath)
-			return nil, err
-		}
-		// Extract base64 part from data URL (e.g., "data:image/jpeg;base64,...")
-		parts := strings.SplitN(imageURL, ",", 2)
-		if len(parts) == 2 {
-			multimodalData = append(multimodalData, parts[1])
-		} else {
-			logger.Error("invalid image data URL format", "url", imageURL)
-			return nil, errors.New("invalid image data URL format")
-		}
-		imageAttachmentPath = "" // Clear the attachment after use
-	}
 	if msg != "" { // otherwise let the bot to continue
-		newMsg := models.RoleMsg{Role: role, Content: msg}
+		var newMsg models.RoleMsg
+		if localImageAttachmentPath != "" {
+			newMsg = models.NewMultimodalMsg(role, []any{})
+			newMsg.AddTextPart(msg)
+			imageURL, err := models.CreateImageURLFromPath(localImageAttachmentPath)
+			if err != nil {
+				logger.Error("failed to create image URL from path for completion",
+					"error", err, "path", localImageAttachmentPath)
+				return nil, err
+			}
+			newMsg.AddImagePart(imageURL, localImageAttachmentPath)
+			imageAttachmentPath = "" // Clear the attachment after use
+		} else { // not a multimodal msg or image passed in tool call
+			newMsg = models.RoleMsg{Role: role, Content: msg}
+		}
 		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	// if rag - add as system message to avoid conflicts with tool usage
-	if !resume && cfg.RAGEnabled {
-		um := chatBody.Messages[len(chatBody.Messages)-1].Content
-		logger.Debug("RAG is enabled, preparing RAG context", "user_message", um)
-		ragResp, err := chatRagUse(um)
-		if err != nil {
-			logger.Error("failed to form a rag msg", "error", err)
-			return nil, err
-		}
-		logger.Debug("RAG response received", "response_len", len(ragResp),
-			"response_preview", ragResp[:min(len(ragResp), 100)])
-		// Use system role for RAG context to avoid conflicts with tool usage
-		ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-		chatBody.Messages = append(chatBody.Messages, ragMsg)
-		logger.Debug("RAG message added to chat body", "message_count", len(chatBody.Messages))
-	}
 	// sending description of the tools and how to use them
 	if cfg.ToolUse && !resume && role == cfg.UserRole && !containsToolSysMsg() {
 		chatBody.Messages = append(chatBody.Messages, models.RoleMsg{Role: cfg.ToolRole, Content: toolSysMsg})
 	}
 	filteredMessages, botPersona := filterMessagesForCurrentCharacter(chatBody.Messages)
+	// Build prompt and extract images inline as we process each message
 	messages := make([]string, len(filteredMessages))
-	for i, m := range filteredMessages {
-		messages[i] = stripThinkingFromMsg(&m).ToPrompt()
+	for i := range filteredMessages {
+		m := stripThinkingFromMsg(&filteredMessages[i])
+		messages[i] = m.ToPrompt()
+		// Extract images from this message and add marker inline
+		if len(m.ContentParts) > 0 {
+			for _, part := range m.ContentParts {
+				var imgURL string
+				// Check for struct type
+				if imgPart, ok := part.(models.ImageContentPart); ok {
+					imgURL = imgPart.ImageURL.URL
+				} else if partMap, ok := part.(map[string]any); ok {
+					// Check for map type (from JSON unmarshaling)
+					if partType, exists := partMap["type"]; exists && partType == "image_url" {
+						if imgURLMap, ok := partMap["image_url"].(map[string]any); ok {
+							if url, ok := imgURLMap["url"].(string); ok {
+								imgURL = url
+							}
+						}
+					}
+				}
+				if imgURL != "" {
+					// Extract base64 part from data URL (e.g., "data:image/jpeg;base64,...")
+					parts := strings.SplitN(imgURL, ",", 2)
+					if len(parts) == 2 {
+						multimodalData = append(multimodalData, parts[1])
+						messages[i] += " <__media__>"
+					}
+				}
+			}
+		}
 	}
 	prompt := strings.Join(messages, "\n")
-	// Add multimodal media markers to the prompt text when multimodal data is present
-	// This is required by llama.cpp multimodal models so they know where to insert media
-	if len(multimodalData) > 0 {
-		// Add a media marker for each item in the multimodal data
-		var sb strings.Builder
-		sb.WriteString(prompt)
-		for range multimodalData {
-			sb.WriteString(" <__media__>") // llama.cpp default multimodal marker
-		}
-		prompt = sb.String()
-	}
 	// needs to be after <__media__> if there are images
 	if !resume {
 		botMsgStart := "\n" + botPersona + ":\n"
 		prompt += botMsgStart
 	}
-	if cfg.ThinkUse && !cfg.ToolUse {
-		prompt += "<think>"
-	}
 	logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
 		"msg", msg, "resume", resume, "prompt", prompt, "multimodal_data_count", len(multimodalData))
 	payload := models.NewLCPReq(prompt, chatBody.Model, multimodalData,
@@ -236,13 +230,11 @@ func (op LCPChat) ParseChunk(data []byte) (*models.TextChunk, error) {
 		logger.Warn("LCPChat ParseChunk: no choices in response", "data", string(data))
 		return &models.TextChunk{Finished: true}, nil
 	}
-
 	lastChoice := llmchunk.Choices[len(llmchunk.Choices)-1]
 	resp := &models.TextChunk{
 		Chunk:     lastChoice.Delta.Content,
 		Reasoning: lastChoice.Delta.ReasoningContent,
 	}
-
 	// Check for tool calls in all choices, not just the last one
 	for _, choice := range llmchunk.Choices {
 		if len(choice.Delta.ToolCalls) > 0 {
@@ -257,7 +249,6 @@ func (op LCPChat) ParseChunk(data []byte) (*models.TextChunk, error) {
 			break // Process only the first tool call
 		}
 	}
-
 	if lastChoice.FinishReason == "stop" {
 		if resp.Chunk != "" {
 			logger.Error("text inside of finish llmchunk", "chunk", llmchunk)
@@ -304,23 +295,6 @@ func (op LCPChat) FormMsg(msg, role string, resume bool) (io.Reader, error) {
 		logger.Debug("LCPChat FormMsg: added message to chatBody", "role", newMsg.Role,
 			"content_len", len(newMsg.Content), "message_count_after_add", len(chatBody.Messages))
 	}
-	// if rag - add as system message to avoid conflicts with tool usage
-	if !resume && cfg.RAGEnabled {
-		um := chatBody.Messages[len(chatBody.Messages)-1].Content
-		logger.Debug("LCPChat: RAG is enabled, preparing RAG context", "user_message", um)
-		ragResp, err := chatRagUse(um)
-		if err != nil {
-			logger.Error("LCPChat: failed to form a rag msg", "error", err)
-			return nil, err
-		}
-		logger.Debug("LCPChat: RAG response received",
-			"response_len", len(ragResp), "response_preview", ragResp[:min(len(ragResp), 100)])
-		// Use system role for RAG context to avoid conflicts with tool usage
-		ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-		chatBody.Messages = append(chatBody.Messages, ragMsg)
-		logger.Debug("LCPChat: RAG message added to chat body", "role", ragMsg.Role,
-			"rag_content_len", len(ragMsg.Content), "message_count_after_rag", len(chatBody.Messages))
-	}
 	filteredMessages, _ := filterMessagesForCurrentCharacter(chatBody.Messages)
 	// openai /v1/chat does not support custom roles; needs to be user, assistant, system
 	// Add persona suffix to the last user message to indicate who the assistant should reply as
@@ -329,14 +303,23 @@ func (op LCPChat) FormMsg(msg, role string, resume bool) (io.Reader, error) {
 		Model:    chatBody.Model,
 		Stream:   chatBody.Stream,
 	}
-	for i, msg := range filteredMessages {
-		strippedMsg := *stripThinkingFromMsg(&msg)
-		if strippedMsg.Role == cfg.UserRole {
+	for i := range filteredMessages {
+		strippedMsg := *stripThinkingFromMsg(&filteredMessages[i])
+		switch strippedMsg.Role {
+		case cfg.UserRole:
 			bodyCopy.Messages[i] = strippedMsg
 			bodyCopy.Messages[i].Role = "user"
-		} else {
+		case cfg.AssistantRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "assistant"
+		case cfg.ToolRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "tool"
+		default:
 			bodyCopy.Messages[i] = strippedMsg
 		}
+		// Clear ToolCalls - they're stored in chat history for display but not sent to LLM
+		// bodyCopy.Messages[i].ToolCall = nil
 	}
 	// Clean null/empty messages to prevent API issues
 	bodyCopy.Messages = consolidateAssistantMessages(bodyCopy.Messages)
@@ -392,30 +375,14 @@ func (ds DeepSeekerCompletion) FormMsg(msg, role string, resume bool) (io.Reader
 		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	// if rag - add as system message to avoid conflicts with tool usage
-	if !resume && cfg.RAGEnabled {
-		um := chatBody.Messages[len(chatBody.Messages)-1].Content
-		logger.Debug("DeepSeekerCompletion: RAG is enabled, preparing RAG context", "user_message", um)
-		ragResp, err := chatRagUse(um)
-		if err != nil {
-			logger.Error("DeepSeekerCompletion: failed to form a rag msg", "error", err)
-			return nil, err
-		}
-		logger.Debug("DeepSeekerCompletion: RAG response received",
-			"response_len", len(ragResp), "response_preview", ragResp[:min(len(ragResp), 100)])
-		// Use system role for RAG context to avoid conflicts with tool usage
-		ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-		chatBody.Messages = append(chatBody.Messages, ragMsg)
-		logger.Debug("DeepSeekerCompletion: RAG message added to chat body", "message_count", len(chatBody.Messages))
-	}
 	// sending description of the tools and how to use them
 	if cfg.ToolUse && !resume && role == cfg.UserRole && !containsToolSysMsg() {
 		chatBody.Messages = append(chatBody.Messages, models.RoleMsg{Role: cfg.ToolRole, Content: toolSysMsg})
 	}
 	filteredMessages, botPersona := filterMessagesForCurrentCharacter(chatBody.Messages)
 	messages := make([]string, len(filteredMessages))
-	for i, m := range filteredMessages {
-		messages[i] = stripThinkingFromMsg(&m).ToPrompt()
+	for i := range filteredMessages {
+		messages[i] = stripThinkingFromMsg(&filteredMessages[i]).ToPrompt()
 	}
 	prompt := strings.Join(messages, "\n")
 	// strings builder?
@@ -423,9 +390,6 @@ func (ds DeepSeekerCompletion) FormMsg(msg, role string, resume bool) (io.Reader
 		botMsgStart := "\n" + botPersona + ":\n"
 		prompt += botMsgStart
 	}
-	if cfg.ThinkUse && !cfg.ToolUse {
-		prompt += "<think>"
-	}
 	logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
 		"msg", msg, "resume", resume, "prompt", prompt)
 	payload := models.NewDSCompletionReq(prompt, chatBody.Model,
@@ -480,22 +444,6 @@ func (ds DeepSeekerChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
 		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	// if rag - add as system message to avoid conflicts with tool usage
-	if !resume && cfg.RAGEnabled {
-		um := chatBody.Messages[len(chatBody.Messages)-1].Content
-		logger.Debug("RAG is enabled, preparing RAG context", "user_message", um)
-		ragResp, err := chatRagUse(um)
-		if err != nil {
-			logger.Error("failed to form a rag msg", "error", err)
-			return nil, err
-		}
-		logger.Debug("RAG response received", "response_len", len(ragResp),
-			"response_preview", ragResp[:min(len(ragResp), 100)])
-		// Use system role for RAG context to avoid conflicts with tool usage
-		ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-		chatBody.Messages = append(chatBody.Messages, ragMsg)
-		logger.Debug("RAG message added to chat body", "message_count", len(chatBody.Messages))
-	}
 	// Create copy of chat body with standardized user role
 	filteredMessages, _ := filterMessagesForCurrentCharacter(chatBody.Messages)
 	// Add persona suffix to the last user message to indicate who the assistant should reply as
@@ -504,14 +452,27 @@ func (ds DeepSeekerChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
 		Model:    chatBody.Model,
 		Stream:   chatBody.Stream,
 	}
-	for i, msg := range filteredMessages {
-		strippedMsg := *stripThinkingFromMsg(&msg)
-		if strippedMsg.Role == cfg.UserRole || i == 1 {
+	for i := range filteredMessages {
+		strippedMsg := *stripThinkingFromMsg(&filteredMessages[i])
+		switch strippedMsg.Role {
+		case cfg.UserRole:
+			if i == 1 {
+				bodyCopy.Messages[i] = strippedMsg
+				bodyCopy.Messages[i].Role = "user"
+			} else {
+				bodyCopy.Messages[i] = strippedMsg
+			}
+		case cfg.AssistantRole:
 			bodyCopy.Messages[i] = strippedMsg
-			bodyCopy.Messages[i].Role = "user"
-		} else {
+			bodyCopy.Messages[i].Role = "assistant"
+		case cfg.ToolRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "tool"
+		default:
 			bodyCopy.Messages[i] = strippedMsg
 		}
+		// Clear ToolCalls - they're stored in chat history for display but not sent to LLM
+		// bodyCopy.Messages[i].ToolCall = nil
 	}
 	// Clean null/empty messages to prevent API issues
 	bodyCopy.Messages = consolidateAssistantMessages(bodyCopy.Messages)
@@ -558,30 +519,14 @@ func (or OpenRouterCompletion) FormMsg(msg, role string, resume bool) (io.Reader
 		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	// if rag - add as system message to avoid conflicts with tool usage
-	if !resume && cfg.RAGEnabled {
-		um := chatBody.Messages[len(chatBody.Messages)-1].Content
-		logger.Debug("RAG is enabled, preparing RAG context", "user_message", um)
-		ragResp, err := chatRagUse(um)
-		if err != nil {
-			logger.Error("failed to form a rag msg", "error", err)
-			return nil, err
-		}
-		logger.Debug("RAG response received", "response_len",
-			len(ragResp), "response_preview", ragResp[:min(len(ragResp), 100)])
-		// Use system role for RAG context to avoid conflicts with tool usage
-		ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-		chatBody.Messages = append(chatBody.Messages, ragMsg)
-		logger.Debug("RAG message added to chat body", "message_count", len(chatBody.Messages))
-	}
 	// sending description of the tools and how to use them
 	if cfg.ToolUse && !resume && role == cfg.UserRole && !containsToolSysMsg() {
 		chatBody.Messages = append(chatBody.Messages, models.RoleMsg{Role: cfg.ToolRole, Content: toolSysMsg})
 	}
 	filteredMessages, botPersona := filterMessagesForCurrentCharacter(chatBody.Messages)
 	messages := make([]string, len(filteredMessages))
-	for i, m := range filteredMessages {
-		messages[i] = stripThinkingFromMsg(&m).ToPrompt()
+	for i := range filteredMessages {
+		messages[i] = stripThinkingFromMsg(&filteredMessages[i]).ToPrompt()
 	}
 	prompt := strings.Join(messages, "\n")
 	// strings builder?
@@ -589,9 +534,6 @@ func (or OpenRouterCompletion) FormMsg(msg, role string, resume bool) (io.Reader
 		botMsgStart := "\n" + botPersona + ":\n"
 		prompt += botMsgStart
 	}
-	if cfg.ThinkUse && !cfg.ToolUse {
-		prompt += "<think>"
-	}
 	stopSlice := chatBody.MakeStopSliceExcluding("", listChatRoles())
 	logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
 		"msg", msg, "resume", resume, "prompt", prompt, "stop_strings", stopSlice)
@@ -679,22 +621,6 @@ func (or OpenRouterChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
 		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	// if rag - add as system message to avoid conflicts with tool usage
-	if !resume && cfg.RAGEnabled {
-		um := chatBody.Messages[len(chatBody.Messages)-1].Content
-		logger.Debug("RAG is enabled, preparing RAG context", "user_message", um)
-		ragResp, err := chatRagUse(um)
-		if err != nil {
-			logger.Error("failed to form a rag msg", "error", err)
-			return nil, err
-		}
-		logger.Debug("RAG response received", "response_len", len(ragResp),
-			"response_preview", ragResp[:min(len(ragResp), 100)])
-		// Use system role for RAG context to avoid conflicts with tool usage
-		ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-		chatBody.Messages = append(chatBody.Messages, ragMsg)
-		logger.Debug("RAG message added to chat body", "message_count", len(chatBody.Messages))
-	}
 	// Create copy of chat body with standardized user role
 	filteredMessages, _ := filterMessagesForCurrentCharacter(chatBody.Messages)
 	// Add persona suffix to the last user message to indicate who the assistant should reply as
@@ -703,14 +629,24 @@ func (or OpenRouterChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
 		Model:    chatBody.Model,
 		Stream:   chatBody.Stream,
 	}
-	for i, msg := range filteredMessages {
-		strippedMsg := *stripThinkingFromMsg(&msg)
-		bodyCopy.Messages[i] = strippedMsg
-		// Standardize role if it's a user role
-		if bodyCopy.Messages[i].Role == cfg.UserRole {
+	for i := range filteredMessages {
+		strippedMsg := *stripThinkingFromMsg(&filteredMessages[i])
+		switch strippedMsg.Role {
+		case cfg.UserRole:
 			bodyCopy.Messages[i] = strippedMsg
 			bodyCopy.Messages[i].Role = "user"
+		case cfg.AssistantRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "assistant"
+		case cfg.ToolRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "tool"
+		default:
+			bodyCopy.Messages[i] = strippedMsg
 		}
+		// Clear ToolCalls - they're stored in chat history for display but not sent to LLM
+		// literally deletes data that we need
+		// bodyCopy.Messages[i].ToolCall = nil
 	}
 	// Clean null/empty messages to prevent API issues
 	bodyCopy.Messages = consolidateAssistantMessages(bodyCopy.Messages)

main.go

@@ -7,14 +7,18 @@ import (
 var (
 	boolColors        = map[bool]string{true: "green", false: "red"}
 	botRespMode       = false
+	toolRunningMode   = false
 	editMode          = false
 	roleEditMode      = false
 	injectRole        = true
 	selectedIndex     = int(-1)
 	shellMode         = false
-	thinkingCollapsed = false
-	statusLineTempl   = "help (F12) | [%s:-:b]llm writes[-:-:-] (F6 to interrupt) | chat: [orange:-:b]%s[-:-:-] (F1) | [%s:-:b]tool use[-:-:-] (ctrl+k) | model: [%s:-:b]%s[-:-:-] (ctrl+l) | [%s:-:b]skip LLM resp[-:-:-] (F10)\nAPI: [orange:-:b]%s[-:-:-] (ctrl+v) | writing as: [orange:-:b]%s[-:-:-] (ctrl+q) | bot will write as [orange:-:b]%s[-:-:-] (ctrl+x)"
-	focusSwitcher     = map[tview.Primitive]tview.Primitive{}
+	shellHistory      []string
+	shellHistoryPos   int = -1
+	thinkingCollapsed     = false
+	toolCollapsed         = true
+	statusLineTempl       = "help (F12) | chat: [orange:-:b]%s[-:-:-] (F1) | [%s:-:b]tool use[-:-:-] (ctrl+k) | model: [%s:-:b]%s[-:-:-] (ctrl+l) | [%s:-:b]skip LLM resp[-:-:-] (F10)\nAPI: [orange:-:b]%s[-:-:-] (ctrl+v) | writing as: [orange:-:b]%s[-:-:-] (ctrl+q) | bot will write as [orange:-:b]%s[-:-:-] (ctrl+x)"
+	focusSwitcher         = map[tview.Primitive]tview.Primitive{}
 )
 
 func main() {

main_test.go

@@ -1,42 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"gf-lt/config"
-	"gf-lt/models"
-	"strings"
-	"testing"
-)
-
-func TestRemoveThinking(t *testing.T) {
-	cases := []struct {
-		cb       *models.ChatBody
-		toolMsgs uint8
-	}{
-		{cb: &models.ChatBody{
-			Stream: true,
-			Messages: []models.RoleMsg{
-				{Role: "tool", Content: "should be ommited"},
-				{Role: "system", Content: "should stay"},
-				{Role: "user", Content: "hello, how are you?"},
-				{Role: "assistant", Content: "Oh, hi. <think>I should thank user and continue the conversation</think> I am geat, thank you! How are you?"},
-			},
-		},
-			toolMsgs: uint8(1),
-		},
-	}
-	for i, tc := range cases {
-					t.Run(fmt.Sprintf("run_%d", i), func(t *testing.T) {
-						cfg = &config.Config{ToolRole: "tool"} // Initialize cfg.ToolRole for test
-						mNum := len(tc.cb.Messages)
-						removeThinking(tc.cb)
-						if len(tc.cb.Messages) != mNum-int(tc.toolMsgs) {
-							t.Errorf("failed to delete tools msg %v; expected %d, got %d", tc.cb.Messages, mNum-int(tc.toolMsgs), len(tc.cb.Messages))
-						}
-						for _, msg := range tc.cb.Messages {
-							if strings.Contains(msg.Content, "<think>") {
-								t.Errorf("msg contains think tag; msg: %s\n", msg.Content)
-							}
-						}
-					})	}
-}

models/consts.go

@@ -0,0 +1,13 @@
+package models
+
+const (
+	LoadedMark        = "(loaded) "
+	ToolRespMultyType = "multimodel_content"
+)
+
+type APIType int
+
+const (
+	APITypeChat APIType = iota
+	APITypeCompletion
+)

models/models.go

@@ -5,28 +5,21 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"path/filepath"
 	"strings"
 )
 
-var (
-	// imageBaseDir is the base directory for displaying image paths.
-	// If set, image paths will be shown relative to this directory.
-	imageBaseDir = ""
-)
-
-// SetImageBaseDir sets the base directory for displaying image paths.
-// If dir is empty, full paths will be shown.
-func SetImageBaseDir(dir string) {
-	imageBaseDir = dir
-}
-
 type FuncCall struct {
 	ID   string            `json:"id,omitempty"`
 	Name string            `json:"name"`
 	Args map[string]string `json:"args"`
 }
 
+type ToolCall struct {
+	ID   string `json:"id,omitempty"`
+	Name string `json:"name"`
+	Args string `json:"arguments"`
+}
+
 type LLMResp struct {
 	Choices []struct {
 		FinishReason string `json:"finish_reason"`
@@ -108,40 +101,56 @@ type RoleMsg struct {
 	Role            string         `json:"role"`
 	Content         string         `json:"-"`
 	ContentParts    []any          `json:"-"`
-	ToolCallID      string         `json:"tool_call_id,omitempty"` // For tool response messages
+	ToolCallID      string         `json:"tool_call_id,omitempty"`     // For tool response messages
+	ToolCall        *ToolCall      `json:"tool_call,omitempty"`        // For assistant messages with tool calls
+	IsShellCommand  bool           `json:"is_shell_command,omitempty"` // True for shell command outputs (always shown)
 	KnownTo         []string       `json:"known_to,omitempty"`
 	Stats           *ResponseStats `json:"stats"`
-	hasContentParts bool           // Flag to indicate which content type to marshal
+	HasContentParts bool           // Flag to indicate which content type to marshal
 }
 
 // MarshalJSON implements custom JSON marshaling for RoleMsg
-func (m *RoleMsg) MarshalJSON() ([]byte, error) {
-	if m.hasContentParts {
+//
+//nolint:gocritic
+func (m RoleMsg) MarshalJSON() ([]byte, error) {
+	if m.HasContentParts {
 		// Use structured content format
 		aux := struct {
-			Role       string   `json:"role"`
-			Content    []any    `json:"content"`
-			ToolCallID string   `json:"tool_call_id,omitempty"`
-			KnownTo    []string `json:"known_to,omitempty"`
+			Role           string         `json:"role"`
+			Content        []any          `json:"content"`
+			ToolCallID     string         `json:"tool_call_id,omitempty"`
+			ToolCall       *ToolCall      `json:"tool_call,omitempty"`
+			IsShellCommand bool           `json:"is_shell_command,omitempty"`
+			KnownTo        []string       `json:"known_to,omitempty"`
+			Stats          *ResponseStats `json:"stats,omitempty"`
 		}{
-			Role:       m.Role,
-			Content:    m.ContentParts,
-			ToolCallID: m.ToolCallID,
-			KnownTo:    m.KnownTo,
+			Role:           m.Role,
+			Content:        m.ContentParts,
+			ToolCallID:     m.ToolCallID,
+			ToolCall:       m.ToolCall,
+			IsShellCommand: m.IsShellCommand,
+			KnownTo:        m.KnownTo,
+			Stats:          m.Stats,
 		}
 		return json.Marshal(aux)
 	} else {
 		// Use simple content format
 		aux := struct {
-			Role       string   `json:"role"`
-			Content    string   `json:"content"`
-			ToolCallID string   `json:"tool_call_id,omitempty"`
-			KnownTo    []string `json:"known_to,omitempty"`
+			Role           string         `json:"role"`
+			Content        string         `json:"content"`
+			ToolCallID     string         `json:"tool_call_id,omitempty"`
+			ToolCall       *ToolCall      `json:"tool_call,omitempty"`
+			IsShellCommand bool           `json:"is_shell_command,omitempty"`
+			KnownTo        []string       `json:"known_to,omitempty"`
+			Stats          *ResponseStats `json:"stats,omitempty"`
 		}{
-			Role:       m.Role,
-			Content:    m.Content,
-			ToolCallID: m.ToolCallID,
-			KnownTo:    m.KnownTo,
+			Role:           m.Role,
+			Content:        m.Content,
+			ToolCallID:     m.ToolCallID,
+			ToolCall:       m.ToolCall,
+			IsShellCommand: m.IsShellCommand,
+			KnownTo:        m.KnownTo,
+			Stats:          m.Stats,
 		}
 		return json.Marshal(aux)
 	}
@@ -151,26 +160,35 @@ func (m *RoleMsg) MarshalJSON() ([]byte, error) {
 func (m *RoleMsg) UnmarshalJSON(data []byte) error {
 	// First, try to unmarshal as structured content format
 	var structured struct {
-		Role       string   `json:"role"`
-		Content    []any    `json:"content"`
-		ToolCallID string   `json:"tool_call_id,omitempty"`
-		KnownTo    []string `json:"known_to,omitempty"`
+		Role           string         `json:"role"`
+		Content        []any          `json:"content"`
+		ToolCallID     string         `json:"tool_call_id,omitempty"`
+		ToolCall       *ToolCall      `json:"tool_call,omitempty"`
+		IsShellCommand bool           `json:"is_shell_command,omitempty"`
+		KnownTo        []string       `json:"known_to,omitempty"`
+		Stats          *ResponseStats `json:"stats,omitempty"`
 	}
 	if err := json.Unmarshal(data, &structured); err == nil && len(structured.Content) > 0 {
 		m.Role = structured.Role
 		m.ContentParts = structured.Content
 		m.ToolCallID = structured.ToolCallID
+		m.ToolCall = structured.ToolCall
+		m.IsShellCommand = structured.IsShellCommand
 		m.KnownTo = structured.KnownTo
-		m.hasContentParts = true
+		m.Stats = structured.Stats
+		m.HasContentParts = true
 		return nil
 	}
 
 	// Otherwise, unmarshal as simple content format
 	var simple struct {
-		Role       string   `json:"role"`
-		Content    string   `json:"content"`
-		ToolCallID string   `json:"tool_call_id,omitempty"`
-		KnownTo    []string `json:"known_to,omitempty"`
+		Role           string         `json:"role"`
+		Content        string         `json:"content"`
+		ToolCallID     string         `json:"tool_call_id,omitempty"`
+		ToolCall       *ToolCall      `json:"tool_call,omitempty"`
+		IsShellCommand bool           `json:"is_shell_command,omitempty"`
+		KnownTo        []string       `json:"known_to,omitempty"`
+		Stats          *ResponseStats `json:"stats,omitempty"`
 	}
 	if err := json.Unmarshal(data, &simple); err != nil {
 		return err
@@ -178,79 +196,17 @@ func (m *RoleMsg) UnmarshalJSON(data []byte) error {
 	m.Role = simple.Role
 	m.Content = simple.Content
 	m.ToolCallID = simple.ToolCallID
+	m.ToolCall = simple.ToolCall
+	m.IsShellCommand = simple.IsShellCommand
 	m.KnownTo = simple.KnownTo
-	m.hasContentParts = false
+	m.Stats = simple.Stats
+	m.HasContentParts = false
 	return nil
 }
 
-func (m *RoleMsg) ToText(i int) string {
-	var contentStr string
-	var imageIndicators []string
-	if !m.hasContentParts {
-		contentStr = m.Content
-	} else {
-		var textParts []string
-		for _, part := range m.ContentParts {
-			switch p := part.(type) {
-			case TextContentPart:
-				if p.Type == "text" {
-					textParts = append(textParts, p.Text)
-				}
-			case ImageContentPart:
-				displayPath := p.Path
-				if displayPath == "" {
-					displayPath = "image"
-				} else {
-					displayPath = extractDisplayPath(displayPath)
-				}
-				imageIndicators = append(imageIndicators, fmt.Sprintf("[orange::i][image: %s][-:-:-]", displayPath))
-			case map[string]any:
-				if partType, exists := p["type"]; exists {
-					switch partType {
-					case "text":
-						if textVal, textExists := p["text"]; textExists {
-							if textStr, isStr := textVal.(string); isStr {
-								textParts = append(textParts, textStr)
-							}
-						}
-					case "image_url":
-						var displayPath string
-						if pathVal, pathExists := p["path"]; pathExists {
-							if pathStr, isStr := pathVal.(string); isStr && pathStr != "" {
-								displayPath = extractDisplayPath(pathStr)
-							}
-						}
-						if displayPath == "" {
-							displayPath = "image"
-						}
-						imageIndicators = append(imageIndicators, fmt.Sprintf("[orange::i][image: %s][-:-:-]", displayPath))
-					}
-				}
-			}
-		}
-		contentStr = strings.Join(textParts, " ") + " "
-	}
-	contentStr, _ = strings.CutPrefix(contentStr, m.Role+":")
-	icon := fmt.Sprintf("(%d) <%s>: ", i, m.Role)
-	var finalContent strings.Builder
-	if len(imageIndicators) > 0 {
-		for _, indicator := range imageIndicators {
-			finalContent.WriteString(indicator)
-			finalContent.WriteString("\n")
-		}
-	}
-	finalContent.WriteString(contentStr)
-	if m.Stats != nil {
-		finalContent.WriteString(fmt.Sprintf("\n[gray::i][%d tok, %.1fs, %.1f t/s][-:-:-]",
-			m.Stats.Tokens, m.Stats.Duration, m.Stats.TokensPerSec))
-	}
-	textMsg := fmt.Sprintf("[-:-:b]%s[-:-:-]\n%s\n", icon, finalContent.String())
-	return strings.ReplaceAll(textMsg, "\n\n", "\n")
-}
-
 func (m *RoleMsg) ToPrompt() string {
 	var contentStr string
-	if !m.hasContentParts {
+	if !m.HasContentParts {
 		contentStr = m.Content
 	} else {
 		// For structured content, just take the text parts
@@ -283,7 +239,7 @@ func NewRoleMsg(role, content string) RoleMsg {
 	return RoleMsg{
 		Role:            role,
 		Content:         content,
-		hasContentParts: false,
+		HasContentParts: false,
 	}
 }
 
@@ -292,7 +248,7 @@ func NewMultimodalMsg(role string, contentParts []any) RoleMsg {
 	return RoleMsg{
 		Role:            role,
 		ContentParts:    contentParts,
-		hasContentParts: true,
+		HasContentParts: true,
 	}
 }
 
@@ -301,7 +257,7 @@ func (m *RoleMsg) HasContent() bool {
 	if m.Content != "" {
 		return true
 	}
-	if m.hasContentParts && len(m.ContentParts) > 0 {
+	if m.HasContentParts && len(m.ContentParts) > 0 {
 		return true
 	}
 	return false
@@ -309,7 +265,7 @@ func (m *RoleMsg) HasContent() bool {
 
 // IsContentParts returns true if the message uses structured content parts
 func (m *RoleMsg) IsContentParts() bool {
-	return m.hasContentParts
+	return m.HasContentParts
 }
 
 // GetContentParts returns the content parts of the message
@@ -326,38 +282,98 @@ func (m *RoleMsg) Copy() RoleMsg {
 		ToolCallID:      m.ToolCallID,
 		KnownTo:         m.KnownTo,
 		Stats:           m.Stats,
-		hasContentParts: m.hasContentParts,
+		HasContentParts: m.HasContentParts,
+		ToolCall:        m.ToolCall,
+		IsShellCommand:  m.IsShellCommand,
 	}
 }
 
+// GetText returns the text content of the message, handling both
+// simple Content and multimodal ContentParts formats.
+func (m *RoleMsg) GetText() string {
+	if !m.HasContentParts {
+		return m.Content
+	}
+	var textParts []string
+	for _, part := range m.ContentParts {
+		switch p := part.(type) {
+		case TextContentPart:
+			if p.Type == "text" {
+				textParts = append(textParts, p.Text)
+			}
+		case map[string]any:
+			if partType, exists := p["type"]; exists {
+				if partType == "text" {
+					if textVal, textExists := p["text"]; textExists {
+						if textStr, isStr := textVal.(string); isStr {
+							textParts = append(textParts, textStr)
+						}
+					}
+				}
+			}
+		}
+	}
+	return strings.Join(textParts, " ")
+}
+
+// SetText updates the text content of the message. If the message has
+// ContentParts (multimodal), it updates the text parts while preserving
+// images. If not, it sets the simple Content field.
+func (m *RoleMsg) SetText(text string) {
+	if !m.HasContentParts {
+		m.Content = text
+		return
+	}
+	var newParts []any
+	for _, part := range m.ContentParts {
+		switch p := part.(type) {
+		case TextContentPart:
+			if p.Type == "text" {
+				p.Text = text
+				newParts = append(newParts, p)
+			} else {
+				newParts = append(newParts, p)
+			}
+		case map[string]any:
+			if partType, exists := p["type"]; exists && partType == "text" {
+				p["text"] = text
+				newParts = append(newParts, p)
+			} else {
+				newParts = append(newParts, p)
+			}
+		default:
+			newParts = append(newParts, part)
+		}
+	}
+	m.ContentParts = newParts
+}
+
 // AddTextPart adds a text content part to the message
 func (m *RoleMsg) AddTextPart(text string) {
-	if !m.hasContentParts {
+	if !m.HasContentParts {
 		// Convert to content parts format
 		if m.Content != "" {
 			m.ContentParts = []any{TextContentPart{Type: "text", Text: m.Content}}
 		} else {
 			m.ContentParts = []any{}
 		}
-		m.hasContentParts = true
+		m.HasContentParts = true
 	}
-
 	textPart := TextContentPart{Type: "text", Text: text}
 	m.ContentParts = append(m.ContentParts, textPart)
 }
 
 // AddImagePart adds an image content part to the message
 func (m *RoleMsg) AddImagePart(imageURL, imagePath string) {
-	if !m.hasContentParts {
+	if !m.HasContentParts {
 		// Convert to content parts format
 		if m.Content != "" {
 			m.ContentParts = []any{TextContentPart{Type: "text", Text: m.Content}}
 		} else {
 			m.ContentParts = []any{}
 		}
-		m.hasContentParts = true
+		m.HasContentParts = true
 	}
-
 	imagePart := ImageContentPart{
 		Type: "image_url",
 		Path: imagePath, // Store the original file path
@@ -375,7 +391,6 @@ func CreateImageURLFromPath(imagePath string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-
 	// Determine the image format based on file extension
 	var mimeType string
 	switch {
@@ -392,39 +407,12 @@ func CreateImageURLFromPath(imagePath string) (string, error) {
 	default:
 		mimeType = "image/jpeg" // default
 	}
-
 	// Encode to base64
 	encoded := base64.StdEncoding.EncodeToString(data)
-
 	// Create data URL
 	return fmt.Sprintf("data:%s;base64,%s", mimeType, encoded), nil
 }
 
-// extractDisplayPath returns a path suitable for display, potentially relative to imageBaseDir
-func extractDisplayPath(p string) string {
-	if p == "" {
-		return ""
-	}
-
-	// If base directory is set, try to make path relative to it
-	if imageBaseDir != "" {
-		if rel, err := filepath.Rel(imageBaseDir, p); err == nil {
-			// Check if relative path doesn't start with ".." (meaning it's within base dir)
-			// If it starts with "..", we might still want to show it as relative
-			// but for now we show full path if it goes outside base dir
-			if !strings.HasPrefix(rel, "..") {
-				p = rel
-			}
-		}
-	}
-
-	// Truncate long paths to last 60 characters if needed
-	if len(p) > 60 {
-		return "..." + p[len(p)-60:]
-	}
-	return p
-}
-
 type ChatBody struct {
 	Model    string    `json:"model"`
 	Stream   bool      `json:"stream"`
@@ -432,16 +420,16 @@ type ChatBody struct {
 }
 
 func (cb *ChatBody) Rename(oldname, newname string) {
-	for i, m := range cb.Messages {
-		cb.Messages[i].Content = strings.ReplaceAll(m.Content, oldname, newname)
-		cb.Messages[i].Role = strings.ReplaceAll(m.Role, oldname, newname)
+	for i := range cb.Messages {
+		cb.Messages[i].Content = strings.ReplaceAll(cb.Messages[i].Content, oldname, newname)
+		cb.Messages[i].Role = strings.ReplaceAll(cb.Messages[i].Role, oldname, newname)
 	}
 }
 
 func (cb *ChatBody) ListRoles() []string {
 	namesMap := make(map[string]struct{})
-	for _, m := range cb.Messages {
-		namesMap[m.Role] = struct{}{}
+	for i := range cb.Messages {
+		namesMap[cb.Messages[i].Role] = struct{}{}
 	}
 	resp := make([]string, len(namesMap))
 	i := 0
@@ -528,24 +516,6 @@ type OpenAIReq struct {
 
 // ===
 
-// type LLMModels struct {
-// 	Object string `json:"object"`
-// 	Data   []struct {
-// 		ID      string `json:"id"`
-// 		Object  string `json:"object"`
-// 		Created int    `json:"created"`
-// 		OwnedBy string `json:"owned_by"`
-// 		Meta    struct {
-// 			VocabType int   `json:"vocab_type"`
-// 			NVocab    int   `json:"n_vocab"`
-// 			NCtxTrain int   `json:"n_ctx_train"`
-// 			NEmbd     int   `json:"n_embd"`
-// 			NParams   int64 `json:"n_params"`
-// 			Size      int64 `json:"size"`
-// 		} `json:"meta"`
-// 	} `json:"data"`
-// }
-
 type LlamaCPPReq struct {
 	Model  string `json:"model"`
 	Stream bool   `json:"stream"`
@@ -638,6 +608,20 @@ func (lcp *LCPModels) ListModels() []string {
 	return resp
 }
 
+func (lcp *LCPModels) HasVision(modelID string) bool {
+	for _, m := range lcp.Data {
+		if m.ID == modelID {
+			args := m.Status.Args
+			for i := 0; i < len(args)-1; i++ {
+				if args[i] == "--mmproj" {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
 type ResponseStats struct {
 	Tokens       int
 	Duration     float64
@@ -651,9 +635,7 @@ type ChatRoundReq struct {
 	Resume  bool
 }
 
-type APIType int
-
-const (
-	APITypeChat APIType = iota
-	APITypeCompletion
-)
+type MultimodalToolResp struct {
+	Type  string              `json:"type"`
+	Parts []map[string]string `json:"parts"`
+}

models/models_test.go

@@ -1,167 +0,0 @@
-package models
-
-import (
-	"strings"
-	"testing"
-)
-
-func TestRoleMsgToTextWithImages(t *testing.T) {
-	tests := []struct {
-		name     string
-		msg      RoleMsg
-		index    int
-		expected string // substring to check
-	}{
-		{
-			name:  "text and image",
-			index: 0,
-			msg: func() RoleMsg {
-				msg := NewMultimodalMsg("user", []interface{}{})
-				msg.AddTextPart("Look at this picture")
-				msg.AddImagePart("data:image/jpeg;base64,abc123", "/home/user/Pictures/cat.jpg")
-				return msg
-			}(),
-			expected: "[orange::i][image: /home/user/Pictures/cat.jpg][-:-:-]",
-		},
-		{
-			name:  "image only",
-			index: 1,
-			msg: func() RoleMsg {
-				msg := NewMultimodalMsg("user", []interface{}{})
-				msg.AddImagePart("data:image/png;base64,xyz789", "/tmp/screenshot_20250217_123456.png")
-				return msg
-			}(),
-			expected: "[orange::i][image: /tmp/screenshot_20250217_123456.png][-:-:-]",
-		},
-		{
-			name:  "long filename truncated",
-			index: 2,
-			msg: func() RoleMsg {
-				msg := NewMultimodalMsg("user", []interface{}{})
-				msg.AddTextPart("Check this")
-				msg.AddImagePart("data:image/jpeg;base64,foo", "/very/long/path/to/a/really_long_filename_that_exceeds_forty_characters.jpg")
-				return msg
-			}(),
-			expected: "[orange::i][image: .../to/a/really_long_filename_that_exceeds_forty_characters.jpg][-:-:-]",
-		},
-		{
-			name:  "multiple images",
-			index: 3,
-			msg: func() RoleMsg {
-				msg := NewMultimodalMsg("user", []interface{}{})
-				msg.AddTextPart("Multiple images")
-				msg.AddImagePart("data:image/jpeg;base64,a", "/path/img1.jpg")
-				msg.AddImagePart("data:image/png;base64,b", "/path/img2.png")
-				return msg
-			}(),
-			expected: "[orange::i][image: /path/img1.jpg][-:-:-]\n[orange::i][image: /path/img2.png][-:-:-]",
-		},
-		{
-			name:  "old format without path",
-			index: 4,
-			msg: RoleMsg{
-				Role:            "user",
-				hasContentParts: true,
-				ContentParts: []interface{}{
-					map[string]interface{}{
-						"type": "image_url",
-						"image_url": map[string]interface{}{
-							"url": "data:image/jpeg;base64,old",
-						},
-					},
-				},
-			},
-			expected: "[orange::i][image: image][-:-:-]",
-		},
-		{
-			name:  "old format with path",
-			index: 5,
-			msg: RoleMsg{
-				Role:            "user",
-				hasContentParts: true,
-				ContentParts: []interface{}{
-					map[string]interface{}{
-						"type": "image_url",
-						"path": "/old/path/photo.jpg",
-						"image_url": map[string]interface{}{
-							"url": "data:image/jpeg;base64,old",
-						},
-					},
-				},
-			},
-			expected: "[orange::i][image: /old/path/photo.jpg][-:-:-]",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			result := tt.msg.ToText(tt.index)
-			if !strings.Contains(result, tt.expected) {
-				t.Errorf("ToText() result does not contain expected indicator\ngot: %s\nwant substring: %s", result, tt.expected)
-			}
-			// Ensure the indicator appears before text content
-			if strings.Contains(tt.expected, "cat.jpg") && strings.Contains(result, "Look at this picture") {
-				indicatorPos := strings.Index(result, "[orange::i][image: /home/user/Pictures/cat.jpg][-:-:-]")
-				textPos := strings.Index(result, "Look at this picture")
-				if indicatorPos == -1 || textPos == -1 || indicatorPos >= textPos {
-					t.Errorf("image indicator should appear before text")
-				}
-			}
-		})
-	}
-}
-
-func TestExtractDisplayPath(t *testing.T) {
-	// Save original base dir
-	originalBaseDir := imageBaseDir
-	defer func() { imageBaseDir = originalBaseDir }()
-
-	tests := []struct {
-		name     string
-		baseDir  string
-		path     string
-		expected string
-	}{
-		{
-			name:     "no base dir shows full path",
-			baseDir:  "",
-			path:     "/home/user/images/cat.jpg",
-			expected: "/home/user/images/cat.jpg",
-		},
-		{
-			name:     "relative path within base dir",
-			baseDir:  "/home/user",
-			path:     "/home/user/images/cat.jpg",
-			expected: "images/cat.jpg",
-		},
-		{
-			name:     "path outside base dir shows full path",
-			baseDir:  "/home/user",
-			path:     "/tmp/test.jpg",
-			expected: "/tmp/test.jpg",
-		},
-		{
-			name:     "same directory",
-			baseDir:  "/home/user/images",
-			path:     "/home/user/images/cat.jpg",
-			expected: "cat.jpg",
-		},
-		{
-			name:     "long path truncated",
-			baseDir:  "",
-			path:     "/very/long/path/to/a/really_long_filename_that_exceeds_sixty_characters_limit_yes_it_is_very_long.jpg",
-			expected: "..._that_exceeds_sixty_characters_limit_yes_it_is_very_long.jpg",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			imageBaseDir = tt.baseDir
-			result := extractDisplayPath(tt.path)
-			if result != tt.expected {
-				t.Errorf("extractDisplayPath(%q) with baseDir=%q = %q, want %q",
-					tt.path, tt.baseDir, result, tt.expected)
-			}
-		})
-	}
-}

models/openrouter.go

@@ -172,3 +172,16 @@ func (orm *ORModels) ListModels(free bool) []string {
 	}
 	return resp
 }
+
+func (orm *ORModels) HasVision(modelID string) bool {
+	for i := range orm.Data {
+		if orm.Data[i].ID == modelID {
+			for _, mod := range orm.Data[i].Architecture.InputModalities {
+				if mod == "image" {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}

models/openrouter_test.go

@@ -62,7 +62,6 @@ func TestORModelsListModels(t *testing.T) {
 			t.Errorf("expected 4 total models, got %d", len(allModels))
 		}
 	})
-
 	t.Run("integration with or_models.json", func(t *testing.T) {
 		// Attempt to load the real data file from the project root
 		path := filepath.Join("..", "or_models.json")

popups.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"gf-lt/models"
 	"slices"
 	"strings"
 
@@ -51,7 +52,7 @@ func showModelSelectionPopup() {
 	// Find the current model index to set as selected
 	currentModelIndex := -1
 	for i, model := range modelList {
-		if model == chatBody.Model {
+		if strings.TrimPrefix(model, models.LoadedMark) == chatBody.Model {
 			currentModelIndex = i
 		}
 		modelListWidget.AddItem(model, "", 0, nil)
@@ -61,20 +62,23 @@ func showModelSelectionPopup() {
 		modelListWidget.SetCurrentItem(currentModelIndex)
 	}
 	modelListWidget.SetSelectedFunc(func(index int, mainText string, secondaryText string, shortcut rune) {
-		modelName := strings.TrimPrefix(mainText, "(loaded) ")
+		modelName := strings.TrimPrefix(mainText, models.LoadedMark)
 		chatBody.Model = modelName
 		cfg.CurrentModel = chatBody.Model
 		pages.RemovePage("modelSelectionPopup")
+		app.SetFocus(textArea)
 		updateCachedModelColor()
 		updateStatusLine()
 	})
 	modelListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("modelSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("modelSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -139,6 +143,7 @@ func showAPILinkSelectionPopup() {
 	apiListWidget.SetSelectedFunc(func(index int, mainText string, secondaryText string, shortcut rune) {
 		// Update the API in config
 		cfg.CurrentAPI = mainText
+		UpdateToolCapabilities()
 		// Update model list based on new API
 		// Helper function to get model list for a given API (same as in props_table.go)
 		getModelListForAPI := func(api string) []string {
@@ -156,10 +161,12 @@ func showAPILinkSelectionPopup() {
 		newModelList := getModelListForAPI(cfg.CurrentAPI)
 		// Ensure chatBody.Model is in the new list; if not, set to first available model
 		if len(newModelList) > 0 && !slices.Contains(newModelList, chatBody.Model) {
-			chatBody.Model = newModelList[0]
+			chatBody.Model = strings.TrimPrefix(newModelList[0], models.LoadedMark)
 			cfg.CurrentModel = chatBody.Model
+			UpdateToolCapabilities()
 		}
 		pages.RemovePage("apiLinkSelectionPopup")
+		app.SetFocus(textArea)
 		choseChunkParser()
 		updateCachedModelColor()
 		updateStatusLine()
@@ -167,10 +174,12 @@ func showAPILinkSelectionPopup() {
 	apiListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("apiLinkSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("apiLinkSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -230,6 +239,7 @@ func showUserRoleSelectionPopup() {
 		textView.SetText(chatToText(filtered, cfg.ShowSys))
 		// Remove the popup page
 		pages.RemovePage("userRoleSelectionPopup")
+		app.SetFocus(textArea)
 		// Update the status line to reflect the change
 		updateStatusLine()
 		colorText()
@@ -237,10 +247,12 @@ func showUserRoleSelectionPopup() {
 	roleListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("userRoleSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("userRoleSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -303,16 +315,19 @@ func showBotRoleSelectionPopup() {
 		cfg.WriteNextMsgAsCompletionAgent = mainText
 		// Remove the popup page
 		pages.RemovePage("botRoleSelectionPopup")
+		app.SetFocus(textArea)
 		// Update the status line to reflect the change
 		updateStatusLine()
 	})
 	roleListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("botRoleSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("botRoleSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -331,7 +346,67 @@ func showBotRoleSelectionPopup() {
 	app.SetFocus(roleListWidget)
 }
 
-func showFileCompletionPopup(filter string) {
+func showShellFileCompletionPopup(filter string) {
+	baseDir := cfg.FilePickerDir
+	if baseDir == "" {
+		baseDir = "."
+	}
+	complMatches := scanFiles(baseDir, filter)
+	if len(complMatches) == 0 {
+		return
+	}
+	if len(complMatches) == 1 {
+		currentText := shellInput.GetText()
+		atIdx := strings.LastIndex(currentText, "@")
+		if atIdx >= 0 {
+			before := currentText[:atIdx]
+			shellInput.SetText(before + complMatches[0])
+		}
+		return
+	}
+	widget := tview.NewList().ShowSecondaryText(false).
+		SetSelectedBackgroundColor(tcell.ColorGray)
+	widget.SetTitle("file completion").SetBorder(true)
+	for _, m := range complMatches {
+		widget.AddItem(m, "", 0, nil)
+	}
+	widget.SetSelectedFunc(func(index int, mainText string, secondaryText string, shortcut rune) {
+		currentText := shellInput.GetText()
+		atIdx := strings.LastIndex(currentText, "@")
+		if atIdx >= 0 {
+			before := currentText[:atIdx]
+			shellInput.SetText(before + mainText)
+		}
+		pages.RemovePage("shellFileCompletionPopup")
+		app.SetFocus(shellInput)
+	})
+	widget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyEscape {
+			pages.RemovePage("shellFileCompletionPopup")
+			app.SetFocus(shellInput)
+			return nil
+		}
+		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
+			pages.RemovePage("shellFileCompletionPopup")
+			app.SetFocus(shellInput)
+			return nil
+		}
+		return event
+	})
+	modal := func(p tview.Primitive, width, height int) tview.Primitive {
+		return tview.NewFlex().
+			AddItem(nil, 0, 1, false).
+			AddItem(tview.NewFlex().SetDirection(tview.FlexRow).
+				AddItem(nil, 0, 1, false).
+				AddItem(p, height, 1, true).
+				AddItem(nil, 0, 1, false), width, 1, true).
+			AddItem(nil, 0, 1, false)
+	}
+	pages.AddPage("shellFileCompletionPopup", modal(widget, 80, 20), true, true)
+	app.SetFocus(widget)
+}
+
+func showTextAreaFileCompletionPopup(filter string) {
 	baseDir := cfg.FilePickerDir
 	if baseDir == "" {
 		baseDir = "."
@@ -340,7 +415,6 @@ func showFileCompletionPopup(filter string) {
 	if len(complMatches) == 0 {
 		return
 	}
-	// If only one match, auto-complete without showing popup
 	if len(complMatches) == 1 {
 		currentText := textArea.GetText()
 		atIdx := strings.LastIndex(currentText, "@")
@@ -363,15 +437,18 @@ func showFileCompletionPopup(filter string) {
 			before := currentText[:atIdx]
 			textArea.SetText(before+mainText, true)
 		}
-		pages.RemovePage("fileCompletionPopup")
+		pages.RemovePage("textAreaFileCompletionPopup")
+		app.SetFocus(textArea)
 	})
 	widget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
-			pages.RemovePage("fileCompletionPopup")
+			pages.RemovePage("textAreaFileCompletionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
-			pages.RemovePage("fileCompletionPopup")
+			pages.RemovePage("textAreaFileCompletionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event
@@ -385,8 +462,7 @@ func showFileCompletionPopup(filter string) {
 				AddItem(nil, 0, 1, false), width, 1, true).
 			AddItem(nil, 0, 1, false)
 	}
-	// Add modal page and make it visible
-	pages.AddPage("fileCompletionPopup", modal(widget, 80, 20), true, true)
+	pages.AddPage("textAreaFileCompletionPopup", modal(widget, 80, 20), true, true)
 	app.SetFocus(widget)
 }
 
@@ -395,38 +471,30 @@ func updateWidgetColors(theme *tview.Theme) {
 	fgColor := theme.PrimaryTextColor
 	borderColor := theme.BorderColor
 	titleColor := theme.TitleColor
-
 	textView.SetBackgroundColor(bgColor)
 	textView.SetTextColor(fgColor)
 	textView.SetBorderColor(borderColor)
 	textView.SetTitleColor(titleColor)
-
 	textArea.SetBackgroundColor(bgColor)
 	textArea.SetBorderColor(borderColor)
 	textArea.SetTitleColor(titleColor)
 	textArea.SetTextStyle(tcell.StyleDefault.Background(bgColor).Foreground(fgColor))
 	textArea.SetPlaceholderStyle(tcell.StyleDefault.Background(bgColor).Foreground(fgColor))
-	// Force textarea refresh by restoring text (SetTextStyle doesn't trigger redraw)
 	textArea.SetText(textArea.GetText(), true)
-
 	editArea.SetBackgroundColor(bgColor)
 	editArea.SetBorderColor(borderColor)
 	editArea.SetTitleColor(titleColor)
 	editArea.SetTextStyle(tcell.StyleDefault.Background(bgColor).Foreground(fgColor))
 	editArea.SetPlaceholderStyle(tcell.StyleDefault.Background(bgColor).Foreground(fgColor))
-	// Force textarea refresh by restoring text (SetTextStyle doesn't trigger redraw)
 	editArea.SetText(editArea.GetText(), true)
-
 	statusLineWidget.SetBackgroundColor(bgColor)
 	statusLineWidget.SetTextColor(fgColor)
 	statusLineWidget.SetBorderColor(borderColor)
 	statusLineWidget.SetTitleColor(titleColor)
-
 	helpView.SetBackgroundColor(bgColor)
 	helpView.SetTextColor(fgColor)
 	helpView.SetBorderColor(borderColor)
 	helpView.SetTitleColor(titleColor)
-
 	searchField.SetBackgroundColor(bgColor)
 	searchField.SetBorderColor(borderColor)
 	searchField.SetTitleColor(titleColor)
@@ -453,7 +521,6 @@ func showColorschemeSelectionPopup() {
 	schemeListWidget := tview.NewList().ShowSecondaryText(false).
 		SetSelectedBackgroundColor(tcell.ColorGray)
 	schemeListWidget.SetTitle("Select Colorscheme").SetBorder(true)
-
 	currentScheme := "default"
 	for name := range colorschemes {
 		if tview.Styles == colorschemes[name] {
@@ -484,14 +551,17 @@ func showColorschemeSelectionPopup() {
 		}
 		// Remove the popup page
 		pages.RemovePage("colorschemeSelectionPopup")
+		app.SetFocus(textArea)
 	})
 	schemeListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if event.Key() == tcell.KeyEscape {
 			pages.RemovePage("colorschemeSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
 			pages.RemovePage("colorschemeSelectionPopup")
+			app.SetFocus(textArea)
 			return nil
 		}
 		return event

props_table.go

@@ -115,9 +115,6 @@ func makePropsTable(props map[string]float32) *tview.Table {
 		row++
 	}
 	// Add checkboxes
-	addCheckboxRow("Insert <think> tag (/completion only)", cfg.ThinkUse, func(checked bool) {
-		cfg.ThinkUse = checked
-	})
 	addCheckboxRow("RAG use", cfg.RAGEnabled, func(checked bool) {
 		cfg.RAGEnabled = checked
 	})

rag/embedder.go

@@ -131,7 +131,6 @@ func (a *APIEmbedder) EmbedSlice(lines []string) ([][]float32, error) {
 		}
 		embeddings[data.Index] = data.Embedding
 	}
-
 	return embeddings, nil
 }
 

rag/extractors.go

@@ -0,0 +1,181 @@
+package rag
+
+import (
+	"archive/zip"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/ledongthuc/pdf"
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/parser"
+	"github.com/yuin/goldmark/renderer/html"
+)
+
+func ExtractText(fpath string) (string, error) {
+	ext := strings.ToLower(path.Ext(fpath))
+	switch ext {
+	case ".txt":
+		return extractTextFromFile(fpath)
+	case ".md", ".markdown":
+		return extractTextFromMarkdown(fpath)
+	case ".html", ".htm":
+		return extractTextFromHtmlFile(fpath)
+	case ".epub":
+		return extractTextFromEpub(fpath)
+	case ".pdf":
+		return extractTextFromPdf(fpath)
+	default:
+		return "", fmt.Errorf("unsupported file format: %s", ext)
+	}
+}
+
+func extractTextFromFile(fpath string) (string, error) {
+	data, err := os.ReadFile(fpath)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
+func extractTextFromHtmlFile(fpath string) (string, error) {
+	data, err := os.ReadFile(fpath)
+	if err != nil {
+		return "", err
+	}
+	return extractTextFromHtmlContent(data)
+}
+
+// non utf-8 encoding?
+func extractTextFromHtmlContent(data []byte) (string, error) {
+	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(data))
+	if err != nil {
+		return "", err
+	}
+	// Remove script and style tags
+	doc.Find("script, style, noscript").Each(func(i int, s *goquery.Selection) {
+		s.Remove()
+	})
+	// Get text and clean it
+	text := doc.Text()
+	// Collapse all whitespace (newlines, tabs, multiple spaces) into single spaces
+	cleaned := strings.Join(strings.Fields(text), " ")
+	return cleaned, nil
+}
+
+func extractTextFromMarkdown(fpath string) (string, error) {
+	data, err := os.ReadFile(fpath)
+	if err != nil {
+		return "", err
+	}
+	// Convert markdown to HTML
+	md := goldmark.New(
+		goldmark.WithExtensions(extension.GFM),
+		goldmark.WithParserOptions(parser.WithAutoHeadingID()),
+		goldmark.WithRendererOptions(html.WithUnsafe()), // allow raw HTML if needed
+	)
+	var buf bytes.Buffer
+	if err := md.Convert(data, &buf); err != nil {
+		return "", err
+	}
+	// Now extract text from the resulting HTML (using goquery or similar)
+	return extractTextFromHtmlContent(buf.Bytes())
+}
+
+func extractTextFromEpub(fpath string) (string, error) {
+	r, err := zip.OpenReader(fpath)
+	if err != nil {
+		return "", fmt.Errorf("failed to open epub: %w", err)
+	}
+	defer r.Close()
+	var sb strings.Builder
+	for _, f := range r.File {
+		ext := strings.ToLower(path.Ext(f.Name))
+		if ext != ".xhtml" && ext != ".html" && ext != ".htm" && ext != ".xml" {
+			continue
+		}
+
+		// Skip manifest, toc, ncx files - they don't contain book content
+		nameLower := strings.ToLower(f.Name)
+		if strings.Contains(nameLower, "toc") || strings.Contains(nameLower, "nav") ||
+			strings.Contains(nameLower, "manifest") || strings.Contains(nameLower, ".opf") ||
+			strings.HasSuffix(nameLower, ".ncx") {
+			continue
+		}
+
+		rc, err := f.Open()
+		if err != nil {
+			continue
+		}
+
+		if sb.Len() > 0 {
+			sb.WriteString("\n\n")
+		}
+		sb.WriteString(f.Name)
+		sb.WriteString("\n")
+
+		buf, readErr := io.ReadAll(rc)
+		rc.Close()
+		if readErr == nil {
+			sb.WriteString(stripHTML(string(buf)))
+		}
+	}
+	if sb.Len() == 0 {
+		return "", errors.New("no content extracted from epub")
+	}
+	return sb.String(), nil
+}
+
+func stripHTML(html string) string {
+	var sb strings.Builder
+	inTag := false
+	for _, r := range html {
+		switch r {
+		case '<':
+			inTag = true
+		case '>':
+			inTag = false
+		default:
+			if !inTag {
+				sb.WriteRune(r)
+			}
+		}
+	}
+	return sb.String()
+}
+
+func extractTextFromPdf(fpath string) (string, error) {
+	_, err := exec.LookPath("pdftotext")
+	if err == nil {
+		out, err := exec.Command("pdftotext", "-layout", fpath, "-").Output()
+		if err == nil && len(out) > 0 {
+			return string(out), nil
+		}
+	}
+	return extractTextFromPdfPureGo(fpath)
+}
+
+func extractTextFromPdfPureGo(fpath string) (string, error) {
+	df, r, err := pdf.Open(fpath)
+	if err != nil {
+		return "", fmt.Errorf("failed to open pdf: %w", err)
+	}
+	defer df.Close()
+	textReader, err := r.GetPlainText()
+	if err != nil {
+		return "", fmt.Errorf("failed to extract text from pdf: %w", err)
+	}
+	var buf bytes.Buffer
+	_, err = io.Copy(&buf, textReader)
+	if err != nil {
+		return "", fmt.Errorf("failed to read pdf text: %w", err)
+	}
+	return buf.String(), nil
+}

rag/rag.go

@@ -7,8 +7,9 @@ import (
 	"gf-lt/models"
 	"gf-lt/storage"
 	"log/slog"
-	"os"
 	"path"
+	"regexp"
+	"sort"
 	"strings"
 	"sync"
 
@@ -23,19 +24,18 @@ var (
 	ErrRAGStatus        = "some error occurred; failed to transfer data to vector db"
 )
 
-
 type RAG struct {
 	logger   *slog.Logger
 	store    storage.FullRepo
 	cfg      *config.Config
 	embedder Embedder
 	storage  *VectorStorage
+	mu       sync.Mutex
 }
 
 func New(l *slog.Logger, s storage.FullRepo, cfg *config.Config) *RAG {
 	// Initialize with API embedder by default, could be configurable later
 	embedder := NewAPIEmbedder(l, cfg)
-
 	rag := &RAG{
 		logger:   l,
 		store:    s,
@@ -54,7 +54,9 @@ func wordCounter(sentence string) int {
 }
 
 func (r *RAG) LoadRAG(fpath string) error {
-	data, err := os.ReadFile(fpath)
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	fileText, err := ExtractText(fpath)
 	if err != nil {
 		return err
 	}
@@ -63,10 +65,7 @@ func (r *RAG) LoadRAG(fpath string) error {
 	case LongJobStatusCh <- LoadedFileRAGStatus:
 	default:
 		r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", LoadedFileRAGStatus)
-		// Channel is full or closed, ignore the message to prevent panic
 	}
-
-	fileText := string(data)
 	tokenizer, err := english.NewSentenceTokenizer(nil)
 	if err != nil {
 		return err
@@ -76,19 +75,16 @@ func (r *RAG) LoadRAG(fpath string) error {
 	for i, s := range sentences {
 		sents[i] = s.Text
 	}
-
 	// Group sentences into paragraphs based on word limit
 	paragraphs := []string{}
 	par := strings.Builder{}
 	for i := 0; i < len(sents); i++ {
-		// Only add sentences that aren't empty
 		if strings.TrimSpace(sents[i]) != "" {
 			if par.Len() > 0 {
-				par.WriteString(" ") // Add space between sentences
+				par.WriteString(" ")
 			}
 			par.WriteString(sents[i])
 		}
-
 		if wordCounter(par.String()) > int(r.cfg.RAGWordLimit) {
 			paragraph := strings.TrimSpace(par.String())
 			if paragraph != "" {
@@ -97,7 +93,6 @@ func (r *RAG) LoadRAG(fpath string) error {
 			par.Reset()
 		}
 	}
-
 	// Handle any remaining content in the paragraph buffer
 	if par.Len() > 0 {
 		paragraph := strings.TrimSpace(par.String())
@@ -105,215 +100,82 @@ func (r *RAG) LoadRAG(fpath string) error {
 			paragraphs = append(paragraphs, paragraph)
 		}
 	}
-
 	// Adjust batch size if needed
 	if len(paragraphs) < r.cfg.RAGBatchSize && len(paragraphs) > 0 {
 		r.cfg.RAGBatchSize = len(paragraphs)
 	}
-
 	if len(paragraphs) == 0 {
 		return errors.New("no valid paragraphs found in file")
 	}
-
-	var (
-		maxChSize = 100
-		left      = 0
-		right     = r.cfg.RAGBatchSize
-		batchCh   = make(chan map[int][]string, maxChSize)
-		vectorCh  = make(chan []models.VectorRow, maxChSize)
-		errCh     = make(chan error, 1)
-		wg        = new(sync.WaitGroup)
-		lock      = new(sync.Mutex)
-	)
-
-	defer close(errCh)
-	defer close(batchCh)
-
-	// Fill input channel with batches
-	ctn := 0
-	totalParagraphs := len(paragraphs)
-	for {
-		if right > totalParagraphs {
-			batchCh <- map[int][]string{left: paragraphs[left:]}
-			break
+	// Process paragraphs in batches synchronously
+	batchCount := 0
+	for i := 0; i < len(paragraphs); i += r.cfg.RAGBatchSize {
+		end := i + r.cfg.RAGBatchSize
+		if end > len(paragraphs) {
+			end = len(paragraphs)
 		}
-		batchCh <- map[int][]string{left: paragraphs[left:right]}
-		left, right = right, right+r.cfg.RAGBatchSize
-		ctn++
-	}
-
-	finishedBatchesMsg := fmt.Sprintf("finished batching batches#: %d; paragraphs: %d; sentences: %d\n", ctn+1, len(paragraphs), len(sents))
-	r.logger.Debug(finishedBatchesMsg)
-	select {
-	case LongJobStatusCh <- finishedBatchesMsg:
-	default:
-		r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", finishedBatchesMsg)
-		// Channel is full or closed, ignore the message to prevent panic
-	}
-
-	// Start worker goroutines with WaitGroup
-	wg.Add(int(r.cfg.RAGWorkers))
-	for w := 0; w < int(r.cfg.RAGWorkers); w++ {
-		go func(workerID int) {
-			defer wg.Done()
-			r.batchToVectorAsync(lock, workerID, batchCh, vectorCh, errCh, path.Base(fpath))
-		}(w)
-	}
-
-	// Use a goroutine to close the batchCh when all batches are sent
-	go func() {
-		wg.Wait()
-		close(vectorCh) // Close vectorCh when all workers are done
-	}()
-
-	// Check for errors from workers
-	// Use a non-blocking check for errors
-	select {
-	case err := <-errCh:
+		batch := paragraphs[i:end]
+		batchCount++
+		// Filter empty paragraphs
+		nonEmptyBatch := make([]string, 0, len(batch))
+		for _, p := range batch {
+			if strings.TrimSpace(p) != "" {
+				nonEmptyBatch = append(nonEmptyBatch, strings.TrimSpace(p))
+			}
+		}
+		if len(nonEmptyBatch) == 0 {
+			continue
+		}
+		// Embed the batch
+		embeddings, err := r.embedder.EmbedSlice(nonEmptyBatch)
 		if err != nil {
-			r.logger.Error("error during RAG processing", "error", err)
+			r.logger.Error("failed to embed batch", "error", err, "batch", batchCount)
+			select {
+			case LongJobStatusCh <- ErrRAGStatus:
+			default:
+				r.logger.Warn("LongJobStatusCh channel full, dropping message")
+			}
+			return fmt.Errorf("failed to embed batch %d: %w", batchCount, err)
+		}
+		if len(embeddings) != len(nonEmptyBatch) {
+			err := errors.New("embedding count mismatch")
+			r.logger.Error("embedding mismatch", "expected", len(nonEmptyBatch), "got", len(embeddings))
 			return err
 		}
-	default:
-		// No immediate error, continue
-	}
-
-	// Write vectors to storage - this will block until vectorCh is closed
-	return r.writeVectors(vectorCh)
-}
-
-func (r *RAG) writeVectors(vectorCh chan []models.VectorRow) error {
-	for {
-		for batch := range vectorCh {
-			for _, vector := range batch {
-				if err := r.storage.WriteVector(&vector); err != nil {
-					r.logger.Error("failed to write vector to DB", "error", err, "slug", vector.Slug)
-					select {
-					case LongJobStatusCh <- ErrRAGStatus:
-					default:
-						r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", ErrRAGStatus)
-						// Channel is full or closed, ignore the message to prevent panic
-					}
-					return err // Stop the entire RAG operation on DB error
-				}
+		// Write vectors to storage
+		filename := path.Base(fpath)
+		for j, text := range nonEmptyBatch {
+			vector := models.VectorRow{
+				Embeddings: embeddings[j],
+				RawText:    text,
+				Slug:       fmt.Sprintf("%s_%d_%d", filename, batchCount, j),
+				FileName:   filename,
 			}
-			r.logger.Debug("wrote batch to db", "size", len(batch), "vector_chan_len", len(vectorCh))
-			if len(vectorCh) == 0 {
-				r.logger.Debug("finished writing vectors")
+			if err := r.storage.WriteVector(&vector); err != nil {
+				r.logger.Error("failed to write vector to DB", "error", err, "slug", vector.Slug)
 				select {
-				case LongJobStatusCh <- FinishedRAGStatus:
+				case LongJobStatusCh <- ErrRAGStatus:
 				default:
-					r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", FinishedRAGStatus)
-					// Channel is full or closed, ignore the message to prevent panic
+					r.logger.Warn("LongJobStatusCh channel full, dropping message")
 				}
-				return nil
+				return fmt.Errorf("failed to write vector: %w", err)
 			}
 		}
-	}
-}
-
-func (r *RAG) batchToVectorAsync(lock *sync.Mutex, id int, inputCh <-chan map[int][]string,
-	vectorCh chan<- []models.VectorRow, errCh chan error, filename string) {
-	var err error
-
-	defer func() {
-		// For errCh, make sure we only send if there's actually an error and the channel can accept it
-		if err != nil {
-			select {
-			case errCh <- err:
-			default:
-				// errCh might be full or closed, log but don't panic
-				r.logger.Warn("errCh channel full or closed, skipping error propagation", "worker", id, "error", err)
-			}
-		}
-	}()
-
-	for {
-		lock.Lock()
-		if len(inputCh) == 0 {
-			lock.Unlock()
-			return
-		}
-
-		select {
-		case linesMap := <-inputCh:
-			for leftI, lines := range linesMap {
-				if err := r.fetchEmb(lines, errCh, vectorCh, fmt.Sprintf("%s_%d", filename, leftI), filename); err != nil {
-					r.logger.Error("error fetching embeddings", "error", err, "worker", id)
-					lock.Unlock()
-					return
-				}
-			}
-			lock.Unlock()
-		case err = <-errCh:
-			r.logger.Error("got an error from error channel", "error", err)
-			lock.Unlock()
-			return
-		default:
-			lock.Unlock()
-		}
-
-		r.logger.Debug("processed batch", "batches#", len(inputCh), "worker#", id)
-		statusMsg := fmt.Sprintf("converted to vector; batches: %d, worker#: %d", len(inputCh), id)
+		r.logger.Debug("wrote batch to db", "batch", batchCount, "size", len(nonEmptyBatch))
+		// Send progress status
+		statusMsg := fmt.Sprintf("processed batch %d/%d", batchCount, (len(paragraphs)+r.cfg.RAGBatchSize-1)/r.cfg.RAGBatchSize)
 		select {
 		case LongJobStatusCh <- statusMsg:
 		default:
-			r.logger.Warn("LongJobStatusCh channel full or closed, dropping status message", "message", statusMsg)
-			// Channel is full or closed, ignore the message to prevent panic
+			r.logger.Warn("LongJobStatusCh channel full, dropping message")
 		}
 	}
-}
-
-func (r *RAG) fetchEmb(lines []string, errCh chan error, vectorCh chan<- []models.VectorRow, slug, filename string) error {
-	// Filter out empty lines before sending to embedder
-	nonEmptyLines := make([]string, 0, len(lines))
-	for _, line := range lines {
-		trimmed := strings.TrimSpace(line)
-		if trimmed != "" {
-			nonEmptyLines = append(nonEmptyLines, trimmed)
-		}
+	r.logger.Debug("finished writing vectors", "batches", batchCount)
+	select {
+	case LongJobStatusCh <- FinishedRAGStatus:
+	default:
+		r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", FinishedRAGStatus)
 	}
-
-	// Skip if no non-empty lines
-	if len(nonEmptyLines) == 0 {
-		// Send empty result but don't error
-		vectorCh <- []models.VectorRow{}
-		return nil
-	}
-
-	embeddings, err := r.embedder.EmbedSlice(nonEmptyLines)
-	if err != nil {
-		r.logger.Error("failed to embed lines", "err", err.Error())
-		errCh <- err
-		return err
-	}
-
-	if len(embeddings) == 0 {
-		err := errors.New("no embeddings returned")
-		r.logger.Error("empty embeddings")
-		errCh <- err
-		return err
-	}
-
-	if len(embeddings) != len(nonEmptyLines) {
-		err := errors.New("mismatch between number of lines and embeddings returned")
-		r.logger.Error("embedding mismatch", "err", err.Error())
-		errCh <- err
-		return err
-	}
-
-	// Create a VectorRow for each line in the batch
-	vectors := make([]models.VectorRow, len(nonEmptyLines))
-	for i, line := range nonEmptyLines {
-		vectors[i] = models.VectorRow{
-			Embeddings: embeddings[i],
-			RawText:    line,
-			Slug:       fmt.Sprintf("%s_%d", slug, i),
-			FileName:   filename,
-		}
-	}
-
-	vectorCh <- vectors
 	return nil
 }
 
@@ -332,3 +194,259 @@ func (r *RAG) ListLoaded() ([]string, error) {
 func (r *RAG) RemoveFile(filename string) error {
 	return r.storage.RemoveEmbByFileName(filename)
 }
+
+var (
+	queryRefinementPattern = regexp.MustCompile(`(?i)(based on my (vector db|vector db|vector database|rags?|past (conversations?|chat|messages?))|from my (files?|documents?|data|information|memory)|search (in|my) (vector db|database|rags?)|rag search for)`)
+	importantKeywords      = []string{"project", "architecture", "code", "file", "chat", "conversation", "topic", "summary", "details", "history", "previous", "my", "user", "me"}
+	stopWords              = []string{"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", "from", "up", "down", "left", "right"}
+)
+
+func (r *RAG) RefineQuery(query string) string {
+	original := query
+	query = strings.TrimSpace(query)
+	if len(query) == 0 {
+		return original
+	}
+	if len(query) <= 3 {
+		return original
+	}
+	query = strings.ToLower(query)
+	for _, stopWord := range stopWords {
+		wordPattern := `\b` + stopWord + `\b`
+		re := regexp.MustCompile(wordPattern)
+		query = re.ReplaceAllString(query, "")
+	}
+	query = strings.TrimSpace(query)
+	if len(query) < 5 {
+		return original
+	}
+	if queryRefinementPattern.MatchString(original) {
+		cleaned := queryRefinementPattern.ReplaceAllString(original, "")
+		cleaned = strings.TrimSpace(cleaned)
+		if len(cleaned) >= 5 {
+			return cleaned
+		}
+	}
+	query = r.extractImportantPhrases(query)
+	if len(query) < 5 {
+		return original
+	}
+	return query
+}
+
+func (r *RAG) extractImportantPhrases(query string) string {
+	words := strings.Fields(query)
+	var important []string
+	for _, word := range words {
+		word = strings.Trim(word, ".,!?;:'\"()[]{}")
+		isImportant := false
+		for _, kw := range importantKeywords {
+			if strings.Contains(strings.ToLower(word), kw) {
+				isImportant = true
+				break
+			}
+		}
+		if isImportant || len(word) > 3 {
+			important = append(important, word)
+		}
+	}
+	if len(important) == 0 {
+		return query
+	}
+	return strings.Join(important, " ")
+}
+
+func (r *RAG) GenerateQueryVariations(query string) []string {
+	variations := []string{query}
+	if len(query) < 5 {
+		return variations
+	}
+	parts := strings.Fields(query)
+	if len(parts) == 0 {
+		return variations
+	}
+	if len(parts) >= 2 {
+		trimmed := strings.Join(parts[:len(parts)-1], " ")
+		if len(trimmed) >= 5 {
+			variations = append(variations, trimmed)
+		}
+	}
+	if len(parts) >= 2 {
+		trimmed := strings.Join(parts[1:], " ")
+		if len(trimmed) >= 5 {
+			variations = append(variations, trimmed)
+		}
+	}
+	if !strings.HasSuffix(query, " explanation") {
+		variations = append(variations, query+" explanation")
+	}
+	if !strings.HasPrefix(query, "what is ") {
+		variations = append(variations, "what is "+query)
+	}
+	if !strings.HasSuffix(query, " details") {
+		variations = append(variations, query+" details")
+	}
+	if !strings.HasSuffix(query, " summary") {
+		variations = append(variations, query+" summary")
+	}
+	return variations
+}
+
+func (r *RAG) RerankResults(results []models.VectorRow, query string) []models.VectorRow {
+	type scoredResult struct {
+		row      models.VectorRow
+		distance float32
+	}
+	scored := make([]scoredResult, 0, len(results))
+	for i := range results {
+		row := results[i]
+
+		score := float32(0)
+		rawTextLower := strings.ToLower(row.RawText)
+		queryLower := strings.ToLower(query)
+		if strings.Contains(rawTextLower, queryLower) {
+			score += 10
+		}
+		queryWords := strings.Fields(queryLower)
+		matchCount := 0
+		for _, word := range queryWords {
+			if len(word) > 2 && strings.Contains(rawTextLower, word) {
+				matchCount++
+			}
+		}
+		if len(queryWords) > 0 {
+			score += float32(matchCount) / float32(len(queryWords)) * 5
+		}
+		if row.FileName == "chat" || strings.Contains(strings.ToLower(row.FileName), "conversation") {
+			score += 3
+		}
+		distance := row.Distance - score/100
+		scored = append(scored, scoredResult{row: row, distance: distance})
+	}
+	sort.Slice(scored, func(i, j int) bool {
+		return scored[i].distance < scored[j].distance
+	})
+	unique := make([]models.VectorRow, 0)
+	seen := make(map[string]bool)
+	for i := range scored {
+		if !seen[scored[i].row.Slug] {
+			seen[scored[i].row.Slug] = true
+			unique = append(unique, scored[i].row)
+		}
+	}
+	if len(unique) > 10 {
+		unique = unique[:10]
+	}
+	return unique
+}
+
+func (r *RAG) SynthesizeAnswer(results []models.VectorRow, query string) (string, error) {
+	if len(results) == 0 {
+		return "No relevant information found in the vector database.", nil
+	}
+	var contextBuilder strings.Builder
+	contextBuilder.WriteString("User Query: ")
+	contextBuilder.WriteString(query)
+	contextBuilder.WriteString("\n\nRetrieved Context:\n")
+	for i, row := range results {
+		fmt.Fprintf(&contextBuilder, "[Source %d: %s]\n", i+1, row.FileName)
+		contextBuilder.WriteString(row.RawText)
+		contextBuilder.WriteString("\n\n")
+	}
+	contextBuilder.WriteString("Instructions: ")
+	contextBuilder.WriteString("Based on the retrieved context above, provide a concise, coherent answer to the user's query. ")
+	contextBuilder.WriteString("Extract only the most relevant information. ")
+	contextBuilder.WriteString("If no relevant information is found, state that clearly. ")
+	contextBuilder.WriteString("Cite sources by filename when relevant. ")
+	contextBuilder.WriteString("Do not include unnecessary preamble or explanations.")
+	synthesisPrompt := contextBuilder.String()
+	emb, err := r.LineToVector(synthesisPrompt)
+	if err != nil {
+		r.logger.Error("failed to embed synthesis prompt", "error", err)
+		return "", err
+	}
+	embResp := &models.EmbeddingResp{
+		Embedding: emb,
+		Index:     0,
+	}
+	topResults, err := r.SearchEmb(embResp)
+	if err != nil {
+		r.logger.Error("failed to search for synthesis context", "error", err)
+		return "", err
+	}
+	if len(topResults) > 0 && topResults[0].RawText != synthesisPrompt {
+		return topResults[0].RawText, nil
+	}
+	var finalAnswer strings.Builder
+	finalAnswer.WriteString("Based on the retrieved context:\n\n")
+	for i, row := range results {
+		if i >= 5 {
+			break
+		}
+		fmt.Fprintf(&finalAnswer, "- From %s: %s\n", row.FileName, truncateString(row.RawText, 200))
+	}
+	return finalAnswer.String(), nil
+}
+
+func truncateString(s string, maxLen int) string {
+	if len(s) <= maxLen {
+		return s
+	}
+	return s[:maxLen] + "..."
+}
+
+func (r *RAG) Search(query string, limit int) ([]models.VectorRow, error) {
+	refined := r.RefineQuery(query)
+	variations := r.GenerateQueryVariations(refined)
+	allResults := make([]models.VectorRow, 0)
+	seen := make(map[string]bool)
+	for _, q := range variations {
+		emb, err := r.LineToVector(q)
+		if err != nil {
+			r.logger.Error("failed to embed query variation", "error", err, "query", q)
+			continue
+		}
+
+		embResp := &models.EmbeddingResp{
+			Embedding: emb,
+			Index:     0,
+		}
+
+		results, err := r.SearchEmb(embResp)
+		if err != nil {
+			r.logger.Error("failed to search embeddings", "error", err, "query", q)
+			continue
+		}
+
+		for _, row := range results {
+			if !seen[row.Slug] {
+				seen[row.Slug] = true
+				allResults = append(allResults, row)
+			}
+		}
+	}
+	reranked := r.RerankResults(allResults, query)
+	if len(reranked) > limit {
+		reranked = reranked[:limit]
+	}
+	return reranked, nil
+}
+
+var (
+	ragInstance *RAG
+	ragOnce     sync.Once
+)
+
+func Init(c *config.Config, l *slog.Logger, s storage.FullRepo) error {
+	ragOnce.Do(func() {
+		if c == nil || l == nil || s == nil {
+			return
+		}
+		ragInstance = New(l, s, c)
+	})
+	return nil
+}
+
+func GetInstance() *RAG {
+	return ragInstance
+}

rag/storage.go

@@ -28,7 +28,6 @@ func NewVectorStorage(logger *slog.Logger, store storage.FullRepo) *VectorStorag
 	}
 }
 
-
 // SerializeVector converts []float32 to binary blob
 func SerializeVector(vec []float32) []byte {
 	buf := make([]byte, len(vec)*4) // 4 bytes per float32
@@ -66,17 +65,14 @@ func (vs *VectorStorage) WriteVector(row *models.VectorRow) error {
 
 	// Serialize the embeddings to binary
 	serializedEmbeddings := SerializeVector(row.Embeddings)
-
 	query := fmt.Sprintf(
 		"INSERT INTO %s (embeddings, slug, raw_text, filename) VALUES (?, ?, ?, ?)",
 		tableName,
 	)
-
 	if _, err := vs.sqlxDB.Exec(query, serializedEmbeddings, row.Slug, row.RawText, row.FileName); err != nil {
 		vs.logger.Error("failed to write vector", "error", err, "slug", row.Slug)
 		return err
 	}
-
 	return nil
 }
 
@@ -86,20 +82,18 @@ func (vs *VectorStorage) getTableName(emb []float32) (string, error) {
 
 	// Check if we support this embedding size
 	supportedSizes := map[int]bool{
-		384:   true,
-		768:   true,
-		1024:  true,
-		1536:  true,
-		2048:  true,
-		3072:  true,
-		4096:  true,
-		5120:  true,
+		384:  true,
+		768:  true,
+		1024: true,
+		1536: true,
+		2048: true,
+		3072: true,
+		4096: true,
+		5120: true,
 	}
-
 	if supportedSizes[size] {
 		return fmt.Sprintf("embeddings_%d", size), nil
 	}
-
 	return "", fmt.Errorf("no table for embedding size of %d", size)
 }
 
@@ -126,9 +120,7 @@ func (vs *VectorStorage) SearchClosest(query []float32) ([]models.VectorRow, err
 		vector   models.VectorRow
 		distance float32
 	}
-
 	var topResults []SearchResult
-
 	// Process vectors one by one to avoid loading everything into memory
 	for rows.Next() {
 		var (
@@ -176,14 +168,12 @@ func (vs *VectorStorage) SearchClosest(query []float32) ([]models.VectorRow, err
 		result.vector.Distance = result.distance
 		results = append(results, result.vector)
 	}
-
 	return results, nil
 }
 
 // ListFiles returns a list of all loaded files
 func (vs *VectorStorage) ListFiles() ([]string, error) {
 	fileLists := make([][]string, 0)
-
 	// Query all supported tables and combine results
 	embeddingSizes := []int{384, 768, 1024, 1536, 2048, 3072, 4096, 5120}
 	for _, size := range embeddingSizes {
@@ -219,14 +209,12 @@ func (vs *VectorStorage) ListFiles() ([]string, error) {
 			}
 		}
 	}
-
 	return allFiles, nil
 }
 
 // RemoveEmbByFileName removes all embeddings associated with a specific filename
 func (vs *VectorStorage) RemoveEmbByFileName(filename string) error {
 	var errors []string
-
 	embeddingSizes := []int{384, 768, 1024, 1536, 2048, 3072, 4096, 5120}
 	for _, size := range embeddingSizes {
 		table := fmt.Sprintf("embeddings_%d", size)
@@ -235,11 +223,9 @@ func (vs *VectorStorage) RemoveEmbByFileName(filename string) error {
 			errors = append(errors, err.Error())
 		}
 	}
-
 	if len(errors) > 0 {
 		return fmt.Errorf("errors occurred: %s", strings.Join(errors, "; "))
 	}
-
 	return nil
 }
 
@@ -248,18 +234,15 @@ func cosineSimilarity(a, b []float32) float32 {
 	if len(a) != len(b) {
 		return 0.0
 	}
-
 	var dotProduct, normA, normB float32
 	for i := 0; i < len(a); i++ {
 		dotProduct += a[i] * b[i]
 		normA += a[i] * a[i]
 		normB += b[i] * b[i]
 	}
-
 	if normA == 0 || normB == 0 {
 		return 0.0
 	}
-
 	return dotProduct / (sqrt(normA) * sqrt(normB))
 }
 
@@ -275,4 +258,3 @@ func sqrt(f float32) float32 {
 	}
 	return guess
 }
-

session.go

@@ -131,13 +131,18 @@ func loadOldChatOrGetNew() []models.RoleMsg {
 	chat, err := store.GetLastChat()
 	if err != nil {
 		logger.Warn("failed to load history chat", "error", err)
+		maxID, err := store.ChatGetMaxID()
+		if err != nil {
+			logger.Error("failed to fetch max chat id", "error", err)
+		}
+		maxID++
 		chat := &models.Chat{
-			ID:        0,
+			ID:        maxID,
 			CreatedAt: time.Now(),
 			UpdatedAt: time.Now(),
 			Agent:     cfg.AssistantRole,
 		}
-		chat.Name = fmt.Sprintf("%s_%v", chat.Agent, chat.CreatedAt.Unix())
+		chat.Name = fmt.Sprintf("%s_%v", chat.Agent, chat.ID)
 		activeChatName = chat.Name
 		chatMap[chat.Name] = chat
 		return defaultStarter
@@ -149,10 +154,6 @@ func loadOldChatOrGetNew() []models.RoleMsg {
 		chatMap[chat.Name] = chat
 		return defaultStarter
 	}
-	// if chat.Name == "" {
-	// 	logger.Warn("empty chat name", "id", chat.ID)
-	// 	chat.Name = fmt.Sprintf("%s_%v", chat.Agent, chat.CreatedAt.Unix())
-	// }
 	chatMap[chat.Name] = chat
 	activeChatName = chat.Name
 	cfg.AssistantRole = chat.Agent

storage/storage.go

@@ -103,7 +103,6 @@ func NewProviderSQL(dbPath string, logger *slog.Logger) FullRepo {
 		return nil
 	}
 	p := ProviderSQL{db: db, logger: logger}
-
 	p.Migrate()
 	return p
 }

storage/vector.go

@@ -73,12 +73,9 @@ func (p ProviderSQL) WriteVector(row *models.VectorRow) error {
 	if err != nil {
 		return err
 	}
-
 	serializedEmbeddings := SerializeVector(row.Embeddings)
-
 	query := fmt.Sprintf("INSERT INTO %s(embeddings, slug, raw_text, filename) VALUES (?, ?, ?, ?)", tableName)
 	_, err = p.db.Exec(query, serializedEmbeddings, row.Slug, row.RawText, row.FileName)
-
 	return err
 }
 
@@ -87,27 +84,22 @@ func (p ProviderSQL) SearchClosest(q []float32) ([]models.VectorRow, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	querySQL := "SELECT embeddings, slug, raw_text, filename FROM " + tableName
 	rows, err := p.db.Query(querySQL)
 	if err != nil {
 		return nil, err
 	}
 	defer rows.Close()
-
 	type SearchResult struct {
 		vector   models.VectorRow
 		distance float32
 	}
-
 	var topResults []SearchResult
-
 	for rows.Next() {
 		var (
-			embeddingsBlob []byte
+			embeddingsBlob          []byte
 			slug, rawText, fileName string
 		)
-
 		if err := rows.Scan(&embeddingsBlob, &slug, &rawText, &fileName); err != nil {
 			continue
 		}
@@ -152,7 +144,6 @@ func (p ProviderSQL) SearchClosest(q []float32) ([]models.VectorRow, error) {
 		result.vector.Distance = result.distance
 		results[i] = result.vector
 	}
-
 	return results, nil
 }
 
@@ -161,18 +152,15 @@ func cosineSimilarity(a, b []float32) float32 {
 	if len(a) != len(b) {
 		return 0.0
 	}
-
 	var dotProduct, normA, normB float32
 	for i := 0; i < len(a); i++ {
 		dotProduct += a[i] * b[i]
 		normA += a[i] * a[i]
 		normB += b[i] * b[i]
 	}
-
 	if normA == 0 || normB == 0 {
 		return 0.0
 	}
-
 	return dotProduct / (sqrt(normA) * sqrt(normB))
 }
 
@@ -229,13 +217,11 @@ func (p ProviderSQL) ListFiles() ([]string, error) {
 			}
 		}
 	}
-
 	return allFiles, nil
 }
 
 func (p ProviderSQL) RemoveEmbByFileName(filename string) error {
 	var errors []string
-
 	tableNames := []string{
 		"embeddings_384", "embeddings_768", "embeddings_1024", "embeddings_1536",
 		"embeddings_2048", "embeddings_3072", "embeddings_4096", "embeddings_5120",
@@ -246,10 +232,8 @@ func (p ProviderSQL) RemoveEmbByFileName(filename string) error {
 			errors = append(errors, err.Error())
 		}
 	}
-
 	if len(errors) > 0 {
 		return fmt.Errorf("errors occurred: %v", errors)
 	}
-
 	return nil
 }

sysprompts/coding_assistant.json

@@ -1,6 +1,6 @@
 {
-  "sys_prompt": "You are an expert software engineering assistant. Your goal is to help users with coding tasks, debugging, refactoring, and software development.\n\n## Core Principles\n1. **Security First**: Never expose secrets, keys, or credentials. Never commit sensitive data.\n2. **No Git Actions**: You can READ git info (status, log, diff) for context, but NEVER perform git actions (commit, add, push, checkout, reset, rm, etc.). Let the user handle all git operations.\n3. **Explore Before Execute**: Always understand the codebase structure before making changes.\n4. **Follow Conventions**: Match existing code style, patterns, and frameworks used in the project.\n5. **Be Concise**: Minimize output tokens while maintaining quality. Avoid unnecessary explanations.\n\n## Workflow for Complex Tasks\nFor multi-step tasks, ALWAYS use the todo system to track progress:\n\n1. **Create Todo List**: At the start of complex tasks, use `todo_create` to break down work into actionable items.\n2. **Update Progress**: Mark items as `in_progress` when working on them, and `completed` when done.\n3. **Check Status**: Use `todo_read` to review your progress.\n\nExample workflow:\n- User: \"Add user authentication to this app\"\n- You: Create todos: [\"Analyze existing auth structure\", \"Check frameworks in use\", \"Implement auth middleware\", \"Add login endpoints\", \"Test implementation\"]\n\n## Task Execution Flow\n\n### Phase 1: Exploration (Always First)\n- Use `file_list` to understand directory structure (path defaults to FilePickerDir if not specified)\n- Use `file_read` to examine relevant files (paths are relative to FilePickerDir unless starting with `/`)\n- Use `execute_command` with `grep`/`find` to search for patterns\n- Check `README` or documentation files\n- Identify: frameworks, conventions, testing approach\n- **Git reads allowed**: You may use `git status`, `git log`, `git diff` for context, but only to inform your work\n- **Path handling**: Relative paths are resolved against FilePickerDir (configurable via Alt+O). Use absolute paths (starting with `/`) to bypass FilePickerDir.\n\n### Phase 2: Planning\n- For complex tasks: create todo items\n- Identify files that need modification\n- Plan your approach following existing patterns\n\n### Phase 3: Implementation\n- Make changes using appropriate file tools\n- Prefer `file_write` for new files, `file_read` then modify for existing files\n- Follow existing code style exactly\n- Use existing libraries and utilities\n\n### Phase 4: Verification\n- Run tests if available (check for test scripts)\n- Run linting/type checking commands\n- Verify changes work as expected\n\n### Phase 5: Completion\n- Update todos to `completed`\n- Provide concise summary of changes\n- Reference specific file paths and line numbers when relevant\n- **DO NOT commit changes** - inform user what was done so they can review and commit themselves\n\n## Tool Usage Guidelines\n\n**File Operations**:\n- `file_read`: Read before editing. Use for understanding code.\n- `file_write`: Overwrite file content completely.\n- `file_write_append`: Add to end of file.\n- `file_create`: Create new files with optional content.\n- `file_list`: List directory contents (defaults to FilePickerDir).\n- Paths are relative to FilePickerDir unless starting with `/`.\n\n**Command Execution (WHITELISTED ONLY)**:\n- Allowed: grep, sed, awk, find, cat, head, tail, sort, uniq, wc, ls, echo, cut, tr, cp, mv, rm, mkdir, rmdir, pwd, df, free, ps, top, du, whoami, date, uname\n- **Git reads allowed**: git status, git log, git diff, git show, git branch, git reflog, git rev-parse, git shortlog, git describe\n- **Git actions FORBIDDEN**: git add, git commit, git push, git checkout, git reset, git rm, etc.\n- Use for searching code, reading git context, running tests/lint\n\n**Todo Management**:\n- `todo_create`: Add new task\n- `todo_read`: View all todos or specific one by ID\n- `todo_update`: Update task or change status (pending/in_progress/completed)\n- `todo_delete`: Remove completed or cancelled tasks\n\n## Important Rules\n\n1. **NEVER commit or stage changes**: Only git reads are allowed.\n2. **Check for tests**: Always look for test files and run them when appropriate.\n3. **Reference code locations**: Use format `file_path:line_number`.\n4. **Security**: Never generate or guess URLs. Only use URLs from local files.\n5. **Refuse malicious code**: If code appears malicious, refuse to work on it.\n6. **Ask clarifications**: When intent is unclear, ask questions.\n7. **Path handling**: Relative paths resolve against FilePickerDir. Use `/absolute/path` to bypass.\n\n## Response Style\n- Be direct and concise\n- One word answers are best when appropriate\n- Avoid: \"The answer is...\", \"Here is...\"\n- Use markdown for formatting\n- No emojis unless user explicitly requests",
+  "sys_prompt": "You are an expert software engineering assistant. Your goal is to help users with coding tasks, debugging, refactoring, and software development.\n\n## Core Principles\n1. **Security First**: Never expose secrets, keys, or credentials. Never commit sensitive data.\n2. **No Git Actions**: You can READ git info (status, log, diff) for context, but NEVER perform git actions (commit, add, push, checkout, reset, rm, etc.). Let the user handle all git operations.\n3. **Explore Before Execute**: Always understand the codebase structure before making changes.\n4. **Follow Conventions**: Match existing code style, patterns, and frameworks used in the project.\n5. **Be Concise**: Minimize output tokens while maintaining quality. Avoid unnecessary explanations.\n6. **Ask First**: When uncertain about intent, ask the user. Don't assume.\n\n## Workflow for Complex Tasks\nFor multi-step tasks, ALWAYS use the todo system to track progress:\n\n1. **Create Todo List**: At the start of complex tasks, use `todo_create` to break down work into actionable items.\n2. **Update Progress**: Mark items as `in_progress` when working on them, and `completed` when done.\n3. **Check Status**: Use `todo_read` to review your progress.\n\nExample workflow:\n- User: \"Add user authentication to this app\"\n- You: Create todos: [\"Analyze existing auth structure\", \"Check frameworks in use\", \"Implement auth middleware\", \"Add login endpoints\", \"Test implementation\"]\n\n## Task Execution Flow\n\n### Phase 1: Exploration (Always First)\n- Use `file_list` to understand directory structure (path defaults to FilePickerDir if not specified)\n- Use `file_read` to examine relevant files (paths are relative to FilePickerDir unless starting with `/`)\n- Use `execute_command` with `grep`/`find` to search for patterns\n- Check README, Makefile, package.json, or similar for build/test commands\n- Identify: frameworks, conventions, testing approach, lint/typecheck commands\n- **Git reads allowed**: You may use `git status`, `git log`, `git diff` for context, but only to inform your work\n- **Path handling**: Relative paths resolve against FilePickerDir; absolute paths (starting with `/`) bypass it\n\n### Phase 2: Planning\n- For complex tasks: create todo items\n- Identify files that need modification\n- Plan your approach following existing patterns\n\n### Phase 3: Implementation\n- Make changes using appropriate file tools\n- Prefer `file_write` for new files, `file_read` then edit for existing files\n- Follow existing code style exactly\n- Use existing libraries and utilities\n\n### Phase 4: Verification\n- Run tests if available (check for test scripts in README/Makefile)\n- Run linting/type checking commands\n- Verify changes work as expected\n\n### Phase 5: Completion\n- Update todos to `completed`\n- Provide concise summary of changes\n- Reference specific file paths and line numbers when relevant\n- **DO NOT commit changes** - inform user what was done so they can review and commit themselves\n\n## Command Execution\n- Use `execute_command` with a single string containing command and arguments (e.g., `go run main.go`, `ls -la`, `cd /tmp`)\n- Use `cd /path` to change the working directory for file operations",
   "role": "CodingAssistant",
   "filepath": "sysprompts/coding_assistant.json",
-  "first_msg": "Hello! I'm your coding assistant. I can help you with software engineering tasks like writing code, debugging, refactoring, and exploring codebases. I work best when you give me specific tasks, and for complex work, I'll create a todo list to track my progress. What would you like to work on?"
+  "first_msg": "Hello! I'm your coding assistant. Give me a specific task and I'll get started. For complex work, I'll track progress with todos."
 }

tables.go

@@ -236,9 +236,59 @@ func makeChatTable(chatMap map[string]models.Chat) *tview.Table {
 }
 
 // nolint:unused
-func makeRAGTable(fileList []string) *tview.Flex {
-	actions := []string{"load", "delete"}
-	rows, cols := len(fileList), len(actions)+1
+func formatSize(size int64) string {
+	units := []string{"B", "KB", "MB", "GB", "TB"}
+	i := 0
+	s := float64(size)
+	for s >= 1024 && i < len(units)-1 {
+		s /= 1024
+		i++
+	}
+	return fmt.Sprintf("%.1f%s", s, units[i])
+}
+
+type ragFileInfo struct {
+	name     string
+	inRAGDir bool
+	isLoaded bool
+	fullPath string
+}
+
+func makeRAGTable(fileList []string, loadedFiles []string) *tview.Flex {
+	// Build set of loaded files for quick lookup
+	loadedSet := make(map[string]bool)
+	for _, f := range loadedFiles {
+		loadedSet[f] = true
+	}
+
+	// Build merged list: files from ragdir + orphaned files from DB
+	ragFiles := make([]ragFileInfo, 0, len(fileList)+len(loadedFiles))
+	seen := make(map[string]bool)
+
+	// Add files from ragdir
+	for _, f := range fileList {
+		ragFiles = append(ragFiles, ragFileInfo{
+			name:     f,
+			inRAGDir: true,
+			isLoaded: loadedSet[f],
+			fullPath: path.Join(cfg.RAGDir, f),
+		})
+		seen[f] = true
+	}
+
+	// Add orphaned files (in DB but not in ragdir)
+	for _, f := range loadedFiles {
+		if !seen[f] {
+			ragFiles = append(ragFiles, ragFileInfo{
+				name:     f,
+				inRAGDir: false,
+				isLoaded: true,
+				fullPath: "",
+			})
+		}
+	}
+	rows := len(ragFiles)
+	cols := 4 // File Name | Preview | Action | Delete
 	fileTable := tview.NewTable().
 		SetBorders(true)
 	longStatusView := tview.NewTextView()
@@ -252,41 +302,92 @@ func makeRAGTable(fileList []string) *tview.Flex {
 		AddItem(fileTable, 0, 60, true)
 	// Add the exit option as the first row (row 0)
 	fileTable.SetCell(0, 0,
-		tview.NewTableCell("Exit RAG manager").
+		tview.NewTableCell("File Name").
 			SetTextColor(tcell.ColorWhite).
 			SetAlign(tview.AlignCenter).
 			SetSelectable(false))
 	fileTable.SetCell(0, 1,
-		tview.NewTableCell("(Close without action)").
-			SetTextColor(tcell.ColorGray).
+		tview.NewTableCell("Preview").
+			SetTextColor(tcell.ColorWhite).
 			SetAlign(tview.AlignCenter).
 			SetSelectable(false))
 	fileTable.SetCell(0, 2,
-		tview.NewTableCell("exit").
-			SetTextColor(tcell.ColorGray).
-			SetAlign(tview.AlignCenter))
+		tview.NewTableCell("Load/Unload").
+			SetTextColor(tcell.ColorWhite).
+			SetAlign(tview.AlignCenter).
+			SetSelectable(false))
+	fileTable.SetCell(0, 3,
+		tview.NewTableCell("Delete").
+			SetTextColor(tcell.ColorWhite).
+			SetAlign(tview.AlignCenter).
+			SetSelectable(false))
 	// Add the file rows starting from row 1
 	for r := 0; r < rows; r++ {
+		f := ragFiles[r]
 		for c := 0; c < cols; c++ {
 			color := tcell.ColorWhite
-			switch {
-			case c < 1:
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell(fileList[r]).
+			switch c {
+			case 0:
+				displayName := f.name
+				if !f.inRAGDir {
+					displayName = f.name + " (orphaned)"
+				}
+				fileTable.SetCell(r+1, c,
+					tview.NewTableCell(displayName).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter).
 						SetSelectable(false))
-			case c == 1: // Action description column - not selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell("(Action)").
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter).
-						SetSelectable(false))
-			default: // Action button column - selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell(actions[c-1]).
+			case 1:
+				if !f.inRAGDir {
+					// Orphaned file - no preview available
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell("not in ragdir").
+							SetTextColor(tcell.ColorYellow).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				} else if fi, err := os.Stat(f.fullPath); err == nil {
+					size := fi.Size()
+					modTime := fi.ModTime()
+					preview := fmt.Sprintf("%s | %s", formatSize(size), modTime.Format("2006-01-02 15:04"))
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell(preview).
+							SetTextColor(color).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				} else {
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell("error").
+							SetTextColor(color).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				}
+			case 2:
+				actionText := "load"
+				if f.isLoaded {
+					actionText = "unload"
+				}
+				if !f.inRAGDir {
+					// Orphaned file - can only unload
+					actionText = "unload"
+				}
+				fileTable.SetCell(r+1, c,
+					tview.NewTableCell(actionText).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter))
+			case 3:
+				if !f.inRAGDir {
+					// Orphaned file - cannot delete from ragdir (not there)
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell("-").
+							SetTextColor(tcell.ColorDarkGray).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				} else {
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell("delete").
+							SetTextColor(color).
+							SetAlign(tview.AlignCenter))
+				}
 			}
 		}
 	}
@@ -318,7 +419,7 @@ func makeRAGTable(fileList []string) *tview.Flex {
 	}()
 	fileTable.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') || key == tcell.KeyCtrlX {
@@ -335,30 +436,58 @@ func makeRAGTable(fileList []string) *tview.Flex {
 		}
 		// defer pages.RemovePage(RAGPage)
 		tc := fileTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		fileTable.SetSelectable(false, false)
 		// Check if the selected row is the exit row (row 0) - do this first to avoid index issues
 		if row == 0 {
 			pages.RemovePage(RAGPage)
 			return
 		}
-		// For file rows, get the filename (row index - 1 because of the exit row at index 0)
-		fpath := fileList[row-1] // -1 to account for the exit row at index 0
-		// notification := fmt.Sprintf("chat: %s; action: %s", fpath, tc.Text)
+		// For file rows, get the file info (row index - 1 because of the exit row at index 0)
+		f := ragFiles[row-1]
+		// Handle "-" case (orphaned file with no delete option)
+		if tc.Text == "-" {
+			pages.RemovePage(RAGPage)
+			return
+		}
 		switch tc.Text {
 		case "load":
-			fpath = path.Join(cfg.RAGDir, fpath)
+			fpath := path.Join(cfg.RAGDir, f.name)
 			longStatusView.SetText("clicked load")
 			go func() {
 				if err := ragger.LoadRAG(fpath); err != nil {
 					logger.Error("failed to embed file", "chat", fpath, "error", err)
 					_ = notifyUser("RAG", "failed to embed file; error: "+err.Error())
-					errCh <- err
-					// pages.RemovePage(RAGPage)
+					app.QueueUpdate(func() {
+						pages.RemovePage(RAGPage)
+					})
 					return
 				}
+				_ = notifyUser("RAG", "file loaded successfully")
+				app.QueueUpdate(func() {
+					pages.RemovePage(RAGPage)
+				})
+			}()
+			return
+		case "unload":
+			longStatusView.SetText("clicked unload")
+			go func() {
+				if err := ragger.RemoveFile(f.name); err != nil {
+					logger.Error("failed to unload file from RAG", "filename", f.name, "error", err)
+					_ = notifyUser("RAG", "failed to unload file; error: "+err.Error())
+					app.QueueUpdate(func() {
+						pages.RemovePage(RAGPage)
+					})
+					return
+				}
+				_ = notifyUser("RAG", "file unloaded successfully")
+				app.QueueUpdate(func() {
+					pages.RemovePage(RAGPage)
+				})
 			}()
 			return
 		case "delete":
-			fpath = path.Join(cfg.RAGDir, fpath)
+			fpath := path.Join(cfg.RAGDir, f.name)
 			if err := os.Remove(fpath); err != nil {
 				logger.Error("failed to delete file", "filename", fpath, "error", err)
 				return
@@ -383,114 +512,6 @@ func makeRAGTable(fileList []string) *tview.Flex {
 	return ragflex
 }
 
-func makeLoadedRAGTable(fileList []string) *tview.Flex {
-	actions := []string{"delete"}
-	rows, cols := len(fileList), len(actions)+1
-	// Add 1 extra row for the "exit" option at the top
-	fileTable := tview.NewTable().
-		SetBorders(true)
-	longStatusView := tview.NewTextView()
-	longStatusView.SetText("Loaded RAG files list")
-	longStatusView.SetBorder(true).SetTitle("status")
-	longStatusView.SetChangedFunc(func() {
-		app.Draw()
-	})
-	ragflex := tview.NewFlex().SetDirection(tview.FlexRow).
-		AddItem(longStatusView, 0, 10, false).
-		AddItem(fileTable, 0, 60, true)
-	// Add the exit option as the first row (row 0)
-	fileTable.SetCell(0, 0,
-		tview.NewTableCell("Exit Loaded Files manager").
-			SetTextColor(tcell.ColorWhite).
-			SetAlign(tview.AlignCenter).
-			SetSelectable(false))
-	fileTable.SetCell(0, 1,
-		tview.NewTableCell("(Close without action)").
-			SetTextColor(tcell.ColorGray).
-			SetAlign(tview.AlignCenter).
-			SetSelectable(false))
-	fileTable.SetCell(0, 2,
-		tview.NewTableCell("exit").
-			SetTextColor(tcell.ColorGray).
-			SetAlign(tview.AlignCenter))
-	// Add the file rows starting from row 1
-	for r := 0; r < rows; r++ {
-		for c := 0; c < cols; c++ {
-			color := tcell.ColorWhite
-			switch {
-			case c < 1:
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell(fileList[r]).
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter).
-						SetSelectable(false))
-			case c == 1: // Action description column - not selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell("(Action)").
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter).
-						SetSelectable(false))
-			default: // Action button column - selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell(actions[c-1]).
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter))
-			}
-		}
-	}
-	fileTable.Select(0, 0).
-		SetFixed(1, 1).
-		SetSelectable(true, false).
-		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
-		SetDoneFunc(func(key tcell.Key) {
-			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') || key == tcell.KeyCtrlX {
-				pages.RemovePage(RAGLoadedPage)
-				return
-			}
-		}).SetSelectedFunc(func(row int, column int) {
-		// If user selects a non-actionable column (0 or 1), move to first action column (2)
-		if column <= 1 {
-			if fileTable.GetColumnCount() > 2 {
-				fileTable.Select(row, 2) // Select first action column
-			}
-			return
-		}
-		tc := fileTable.GetCell(row, column)
-		// Check if the selected row is the exit row (row 0) - do this first to avoid index issues
-		if row == 0 {
-			pages.RemovePage(RAGLoadedPage)
-			return
-		}
-		// For file rows, get the filename (row index - 1 because of the exit row at index 0)
-		fpath := fileList[row-1] // -1 to account for the exit row at index 0
-		switch tc.Text {
-		case "delete":
-			if err := ragger.RemoveFile(fpath); err != nil {
-				logger.Error("failed to delete file from RAG", "filename", fpath, "error", err)
-				longStatusView.SetText(fmt.Sprintf("Error deleting file: %v", err))
-				return
-			}
-			if err := notifyUser("RAG file deleted", fpath+" was deleted from RAG system"); err != nil {
-				logger.Error("failed to send notification", "error", err)
-			}
-			longStatusView.SetText(fpath + " was deleted from RAG system")
-			return
-		default:
-			pages.RemovePage(RAGLoadedPage)
-			return
-		}
-	})
-	// Add input capture to the flex container to handle 'x' key for closing
-	ragflex.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
-		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
-			pages.RemovePage(RAGLoadedPage)
-			return nil
-		}
-		return event
-	})
-	return ragflex
-}
-
 func makeAgentTable(agentList []string) *tview.Table {
 	actions := []string{"filepath", "load"}
 	rows, cols := len(agentList), len(actions)+1
@@ -499,14 +520,14 @@ func makeAgentTable(agentList []string) *tview.Table {
 	for r := 0; r < rows; r++ {
 		for c := 0; c < cols; c++ {
 			color := tcell.ColorWhite
-			switch {
-			case c < 1:
+			switch c {
+			case 0:
 				chatActTable.SetCell(r, c,
 					tview.NewTableCell(agentList[r]).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter).
 						SetSelectable(false))
-			case c == 1:
+			case 1:
 				if actions[c-1] == "filepath" {
 					cc, ok := sysMap[agentList[r]]
 					if !ok {
@@ -533,7 +554,7 @@ func makeAgentTable(agentList []string) *tview.Table {
 	}
 	chatActTable.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') {
@@ -549,6 +570,8 @@ func makeAgentTable(agentList []string) *tview.Table {
 			return
 		}
 		tc := chatActTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		chatActTable.SetSelectable(false, false)
 		selected := agentList[row]
 		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
 		switch tc.Text {
@@ -630,7 +653,7 @@ func makeCodeBlockTable(codeBlocks []string) *tview.Table {
 	}
 	table.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') {
@@ -646,6 +669,8 @@ func makeCodeBlockTable(codeBlocks []string) *tview.Table {
 			return
 		}
 		tc := table.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		table.SetSelectable(false, false)
 		selected := codeBlocks[row]
 		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
 		switch tc.Text {
@@ -702,7 +727,7 @@ func makeImportChatTable(filenames []string) *tview.Table {
 	}
 	chatActTable.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') {
@@ -718,6 +743,8 @@ func makeImportChatTable(filenames []string) *tview.Table {
 			return
 		}
 		tc := chatActTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		chatActTable.SetSelectable(false, false)
 		selected := filenames[row]
 		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
 		switch tc.Text {
@@ -792,6 +819,7 @@ func makeFilePicker() *tview.Flex {
 	// --- NEW: search state ---
 	searching := false
 	searchQuery := ""
+	searchInputMode := false
 	// Helper function to check if a file has an allowed extension from config
 	hasAllowedExtension := func(filename string) bool {
 		if cfg.FilePickerExts == "" {
@@ -984,6 +1012,7 @@ func makeFilePicker() *tview.Flex {
 			case tcell.KeyEsc:
 				// Exit search, clear filter
 				searching = false
+				searchInputMode = false
 				searchQuery = ""
 				refreshList(currentDisplayDir, "")
 				return nil
@@ -993,16 +1022,80 @@ func makeFilePicker() *tview.Flex {
 					refreshList(currentDisplayDir, searchQuery)
 				}
 				return nil
-			case tcell.KeyRune:
-				r := event.Rune()
-				if r != 0 {
-					searchQuery += string(r)
-					refreshList(currentDisplayDir, searchQuery)
+			case tcell.KeyEnter:
+				// Exit search input mode and let normal processing handle selection
+				searchInputMode = false
+				// Get the currently highlighted item in the list
+				itemIndex := listView.GetCurrentItem()
+				if itemIndex >= 0 && itemIndex < listView.GetItemCount() {
+					itemText, _ := listView.GetItemText(itemIndex)
+					// Check for the exit option first
+					if strings.HasPrefix(itemText, "Exit file picker") {
+						pages.RemovePage(filePickerPage)
+						return nil
+					}
+					// Extract the actual filename/directory name by removing the type info
+					actualItemName := itemText
+					if bracketPos := strings.Index(itemText, " ["); bracketPos != -1 {
+						actualItemName = itemText[:bracketPos]
+					}
+					// Check if it's a directory (ends with /)
+					if strings.HasSuffix(actualItemName, "/") {
+						var targetDir string
+						if strings.HasPrefix(actualItemName, "../") {
+							// Parent directory
+							targetDir = path.Dir(currentDisplayDir)
+							if targetDir == currentDisplayDir && currentDisplayDir == "/" {
+								return nil
+							}
+						} else {
+							// Regular subdirectory
+							dirName := strings.TrimSuffix(actualItemName, "/")
+							targetDir = path.Join(currentDisplayDir, dirName)
+						}
+						// Navigate – clear search
+						if cfg.ImagePreview && imgPreview != nil {
+							imgPreview.SetImage(nil)
+						}
+						searching = false
+						searchInputMode = false
+						searchQuery = ""
+						refreshList(targetDir, "")
+						dirStack = append(dirStack, targetDir)
+						currentStackPos = len(dirStack) - 1
+						statusView.SetText("Current: " + targetDir)
+						return nil
+					} else {
+						// It's a file
+						filePath := path.Join(currentDisplayDir, actualItemName)
+						if info, err := os.Stat(filePath); err == nil && !info.IsDir() {
+							if isImageFile(actualItemName) {
+								SetImageAttachment(filePath)
+								statusView.SetText("Image attached: " + filePath + " (will be sent with next message)")
+								pages.RemovePage(filePickerPage)
+							} else {
+								textArea.SetText(filePath, true)
+								app.SetFocus(textArea)
+								pages.RemovePage(filePickerPage)
+							}
+						}
+						return nil
+					}
 				}
 				return nil
+			case tcell.KeyRune:
+				r := event.Rune()
+				if searchInputMode && r != 0 {
+					searchQuery += string(r)
+					refreshList(currentDisplayDir, searchQuery)
+					return nil
+				}
+				// If not in search input mode, pass through for navigation
+				return event
 			default:
-				// Pass all other keys (arrows, Enter, etc.) to normal processing
-				// This allows selecting items while still in search mode
+				// Exit search input mode but keep filter active for navigation
+				searchInputMode = false
+				// Pass all other keys (arrows, etc.) to normal processing
 				return event
 			}
 		}
@@ -1030,41 +1123,18 @@ func makeFilePicker() *tview.Flex {
 			if event.Rune() == '/' {
 				// Enter search mode
 				searching = true
+				searchInputMode = true
 				searchQuery = ""
 				refreshList(currentDisplayDir, "")
 				return nil
 			}
 			if event.Rune() == 's' {
 				// Set FilePickerDir to current directory
-				itemIndex := listView.GetCurrentItem()
-				if itemIndex >= 0 && itemIndex < listView.GetItemCount() {
-					itemText, _ := listView.GetItemText(itemIndex)
-					// Get the actual directory path
-					var targetDir string
-					if strings.HasPrefix(itemText, "Exit") || strings.HasPrefix(itemText, "Select this directory") {
-						targetDir = currentDisplayDir
-					} else {
-						actualItemName := itemText
-						if bracketPos := strings.Index(itemText, " ["); bracketPos != -1 {
-							actualItemName = itemText[:bracketPos]
-						}
-						// nolint: gocritic
-						if strings.HasPrefix(actualItemName, "../") {
-							targetDir = path.Dir(currentDisplayDir)
-						} else if strings.HasSuffix(actualItemName, "/") {
-							dirName := strings.TrimSuffix(actualItemName, "/")
-							targetDir = path.Join(currentDisplayDir, dirName)
-						} else {
-							targetDir = currentDisplayDir
-						}
-					}
-					cfg.FilePickerDir = targetDir
-					if err := notifyUser("FilePickerDir", "Set to: "+targetDir); err != nil {
-						logger.Error("failed to notify user", "error", err)
-					}
-					// pages.RemovePage(filePickerPage)
-					return nil
-				}
+				// Get the actual directory path
+				cfg.FilePickerDir = currentDisplayDir
+				listView.SetTitle("Files & Directories [s: set FilePickerDir]. Current base dir: " + cfg.FilePickerDir)
+				// pages.RemovePage(filePickerPage)
+				return nil
 			}
 		case tcell.KeyEnter:
 			// Get the currently highlighted item in the list

tui.go

@@ -15,6 +15,11 @@ import (
 	"github.com/rivo/tview"
 )
 
+func isFullScreenPageActive() bool {
+	name, _ := pages.GetFrontPage()
+	return name != "main"
+}
+
 var (
 	app                *tview.Application
 	pages              *tview.Pages
@@ -29,6 +34,9 @@ var (
 	indexPickWindow    *tview.InputField
 	renameWindow       *tview.InputField
 	roleEditWindow     *tview.InputField
+	shellInput         *tview.InputField
+	confirmModal       *tview.Modal
+	confirmPageName    = "confirm"
 	fullscreenMode     bool
 	positionVisible    bool = true
 	scrollToEndEnabled bool = true
@@ -74,7 +82,7 @@ var (
 [yellow]Ctrl+p[white]: props edit form (min-p, dry, etc.)
 [yellow]Ctrl+v[white]: show API link selection popup to choose current API
 [yellow]Ctrl+r[white]: start/stop recording from your microphone (needs stt server or whisper binary)
-[yellow]Ctrl+t[white]: remove thinking (<think>) and tool messages from context (delete from chat)
+[yellow]Ctrl+t[white]: (un)collapse tool messages
 [yellow]Ctrl+l[white]: show model selection popup to choose current model
 [yellow]Ctrl+k[white]: switch tool use (recommend tool use to llm after user msg)
 [yellow]Ctrl+a[white]: interrupt tts (needs tts server)
@@ -93,6 +101,7 @@ var (
 [yellow]Alt+8[white]: show char img or last picked img
 [yellow]Alt+9[white]: warm up (load) selected llama.cpp model
 [yellow]Alt+t[white]: toggle thinking blocks visibility (collapse/expand <think> blocks)
+[yellow]Ctrl+t[white]: toggle tool call/response visibility (collapse/expand tool calls and non-shell tool responses)
 [yellow]Alt+i[white]: show colorscheme selection popup
 
 === scrolling chat window (some keys similar to vim) ===
@@ -119,46 +128,111 @@ Press <Enter> or 'x' to return
 `
 )
 
+func setShellMode(enabled bool) {
+	shellMode = enabled
+	go func() {
+		app.QueueUpdateDraw(func() {
+			updateFlexLayout()
+		})
+	}()
+}
+
 func init() {
 	// Start background goroutine to update model color cache
 	startModelColorUpdater()
 	tview.Styles = colorschemes["default"]
 	app = tview.NewApplication()
 	pages = tview.NewPages()
-	textArea = tview.NewTextArea().
-		SetPlaceholder("input is multiline; press <Enter> to start the next line;\npress <Esc> to send the message.")
-	textArea.SetBorder(true).SetTitle("input")
-	// Add input capture for @ completion
-	textArea.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+	shellInput = tview.NewInputField().
+		SetLabel(fmt.Sprintf("[%s]$ ", cfg.FilePickerDir)). // dynamic prompt
+		SetFieldWidth(0).
+		SetDoneFunc(func(key tcell.Key) {
+			if key == tcell.KeyEnter {
+				cmd := shellInput.GetText()
+				if cmd != "" {
+					executeCommandAndDisplay(cmd)
+				}
+				shellInput.SetText("")
+			}
+		})
+	// Copy your file completion logic to shellInput's InputCapture
+	shellInput.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
 		if !shellMode {
 			return event
 		}
-		// Handle Tab key for file completion
-		if event.Key() == tcell.KeyTab {
-			currentText := textArea.GetText()
-			row, col, _, _ := textArea.GetCursor()
-			// Calculate absolute position from row/col
-			lines := strings.Split(currentText, "\n")
-			cursorPos := 0
-			for i := 0; i < row && i < len(lines); i++ {
-				cursorPos += len(lines[i]) + 1 // +1 for newline
-			}
-			cursorPos += col
-			// Look backwards from cursor to find @
-			if cursorPos > 0 {
-				// Find the last @ before cursor
-				textBeforeCursor := currentText[:cursorPos]
-				atIndex := strings.LastIndex(textBeforeCursor, "@")
-				if atIndex >= 0 {
-					// Extract the partial match text after @
-					filter := textBeforeCursor[atIndex+1:]
-					showFileCompletionPopup(filter)
-					return nil // Consume the Tab event
+		// Handle Up arrow for history previous
+		if event.Key() == tcell.KeyUp {
+			if len(shellHistory) > 0 {
+				if shellHistoryPos < len(shellHistory)-1 {
+					shellHistoryPos++
+					shellInput.SetText(shellHistory[len(shellHistory)-1-shellHistoryPos])
 				}
 			}
+			return nil
+		}
+		// Handle Down arrow for history next
+		if event.Key() == tcell.KeyDown {
+			if shellHistoryPos > 0 {
+				shellHistoryPos--
+				shellInput.SetText(shellHistory[len(shellHistory)-1-shellHistoryPos])
+			} else if shellHistoryPos == 0 {
+				shellHistoryPos = -1
+				shellInput.SetText("")
+			}
+			return nil
+		}
+		// Reset history position when user types
+		if event.Key() == tcell.KeyRune {
+			shellHistoryPos = -1
+		}
+		// Handle Tab key for @ file completion
+		if event.Key() == tcell.KeyTab {
+			currentText := shellInput.GetText()
+			atIndex := strings.LastIndex(currentText, "@")
+			if atIndex >= 0 {
+				filter := currentText[atIndex+1:]
+				showShellFileCompletionPopup(filter)
+			}
+			return nil
+		}
+		return event
+	})
+	confirmModal = tview.NewModal().
+		SetText("You are trying to send an empty message.\nIt makes sense if the last message in the chat is from you.\nAre you sure?").
+		AddButtons([]string{"Yes", "No"}).
+		SetButtonBackgroundColor(tcell.ColorBlack).
+		SetButtonTextColor(tcell.ColorWhite).
+		SetDoneFunc(func(buttonIndex int, buttonLabel string) {
+			if buttonLabel == "Yes" {
+				persona := cfg.UserRole
+				if cfg.WriteNextMsgAs != "" {
+					persona = cfg.WriteNextMsgAs
+				}
+				chatRoundChan <- &models.ChatRoundReq{Role: persona, UserMsg: ""}
+			} // In both Yes and No, go back to the main page
+			pages.SwitchToPage("main") // or whatever your main page is named
+		})
+	confirmModal.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyRune {
+			switch event.Rune() {
+			case 'y', 'Y':
+				persona := cfg.UserRole
+				if cfg.WriteNextMsgAs != "" {
+					persona = cfg.WriteNextMsgAs
+				}
+				chatRoundChan <- &models.ChatRoundReq{Role: persona, UserMsg: ""}
+				pages.SwitchToPage("main")
+				return nil
+			case 'n', 'N', 'x', 'X':
+				pages.SwitchToPage("main")
+				return nil
+			}
 		}
 		return event
 	})
+	textArea = tview.NewTextArea().
+		SetPlaceholder("input is multiline; press <Enter> to start the next line;\npress <Esc> to send the message.")
+	textArea.SetBorder(true).SetTitle("input")
 	textView = tview.NewTextView().
 		SetDynamicColors(true).
 		SetRegions(true).
@@ -259,7 +333,7 @@ func init() {
 				pages.RemovePage(editMsgPage)
 				return nil
 			}
-			chatBody.Messages[selectedIndex].Content = editedMsg
+			chatBody.Messages[selectedIndex].SetText(editedMsg)
 			// change textarea
 			textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
 			pages.RemovePage(editMsgPage)
@@ -347,13 +421,14 @@ func init() {
 			case editMode:
 				hideIndexBar() // Hide overlay first
 				pages.AddPage(editMsgPage, editArea, true, true)
-				editArea.SetText(m.Content, true)
+				editArea.SetText(m.GetText(), true)
 			default:
-				if err := copyToClipboard(m.Content); err != nil {
+				msgText := m.GetText()
+				if err := copyToClipboard(msgText); err != nil {
 					logger.Error("failed to copy to clipboard", "error", err)
 				}
-				previewLen := min(30, len(m.Content))
-				notification := fmt.Sprintf("msg '%s' was copied to the clipboard", m.Content[:previewLen])
+				previewLen := min(30, len(msgText))
+				notification := fmt.Sprintf("msg '%s' was copied to the clipboard", msgText[:previewLen])
 				if err := notifyUser("copied", notification); err != nil {
 					logger.Error("failed to send notification", "error", err)
 				}
@@ -524,7 +599,24 @@ func init() {
 			}
 			return nil
 		}
+		// Handle Ctrl+T to toggle tool call/response visibility
+		if event.Key() == tcell.KeyCtrlT {
+			toolCollapsed = !toolCollapsed
+			textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
+			colorText()
+			status := "expanded"
+			if toolCollapsed {
+				status = "collapsed"
+			}
+			if err := notifyUser("tools", "Tool calls/responses "+status); err != nil {
+				logger.Error("failed to send notification", "error", err)
+			}
+			return nil
+		}
 		if event.Key() == tcell.KeyRune && event.Rune() == 'i' && event.Modifiers()&tcell.ModAlt != 0 {
+			if isFullScreenPageActive() {
+				return event
+			}
 			showColorschemeSelectionPopup()
 			return nil
 		}
@@ -634,17 +726,19 @@ func init() {
 		if event.Key() == tcell.KeyF6 {
 			interruptResp = true
 			botRespMode = false
+			toolRunningMode = false
 			return nil
 		}
 		if event.Key() == tcell.KeyF7 {
 			// copy msg to clipboard
 			editMode = false
 			m := chatBody.Messages[len(chatBody.Messages)-1]
-			if err := copyToClipboard(m.Content); err != nil {
+			msgText := m.GetText()
+			if err := copyToClipboard(msgText); err != nil {
 				logger.Error("failed to copy to clipboard", "error", err)
 			}
-			previewLen := min(30, len(m.Content))
-			notification := fmt.Sprintf("msg '%s' was copied to the clipboard", m.Content[:previewLen])
+			previewLen := min(30, len(msgText))
+			notification := fmt.Sprintf("msg '%s' was copied to the clipboard", msgText[:previewLen])
 			if err := notifyUser("copied", notification); err != nil {
 				logger.Error("failed to send notification", "error", err)
 			}
@@ -731,19 +825,17 @@ func init() {
 			return nil
 		}
 		if event.Key() == tcell.KeyCtrlL {
+			if isFullScreenPageActive() {
+				return event
+			}
 			// Show model selection popup instead of rotating models
 			showModelSelectionPopup()
 			return nil
 		}
-		if event.Key() == tcell.KeyCtrlT {
-			// clear context
-			// remove tools and thinking
-			removeThinking(chatBody)
-			textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
-			colorText()
-			return nil
-		}
 		if event.Key() == tcell.KeyCtrlV {
+			if isFullScreenPageActive() {
+				return event
+			}
 			// Show API link selection popup instead of rotating APIs
 			showAPILinkSelectionPopup()
 			return nil
@@ -833,7 +925,7 @@ func init() {
 				// Stop any currently playing TTS first
 				TTSDoneChan <- true
 				lastMsg := chatBody.Messages[len(chatBody.Messages)-1]
-				cleanedText := models.CleanText(lastMsg.Content)
+				cleanedText := models.CleanText(lastMsg.GetText())
 				if cleanedText != "" {
 					// nolint: errcheck
 					go orator.Speak(cleanedText)
@@ -850,11 +942,17 @@ func init() {
 			return nil
 		}
 		if event.Key() == tcell.KeyCtrlQ {
+			if isFullScreenPageActive() {
+				return event
+			}
 			// Show user role selection popup instead of cycling through roles
 			showUserRoleSelectionPopup()
 			return nil
 		}
 		if event.Key() == tcell.KeyCtrlX {
+			if isFullScreenPageActive() {
+				return event
+			}
 			// Show bot role selection popup instead of cycling through roles
 			showBotRoleSelectionPopup()
 			return nil
@@ -893,6 +991,7 @@ func init() {
 					return nil
 				}
 			}
+			// Get files from ragdir
 			fileList := []string{}
 			for _, f := range files {
 				if f.IsDir() {
@@ -900,22 +999,14 @@ func init() {
 				}
 				fileList = append(fileList, f.Name())
 			}
-			chatRAGTable := makeRAGTable(fileList)
-			pages.AddPage(RAGPage, chatRAGTable, true, true)
-			return nil
-		}
-		if event.Key() == tcell.KeyCtrlY { // Use Ctrl+Y to list loaded RAG files
-			// List files already loaded into the RAG system
-			fileList, err := ragger.ListLoaded()
+			// Get loaded files from vector DB
+			loadedFiles, err := ragger.ListLoaded()
 			if err != nil {
 				logger.Error("failed to list loaded RAG files", "error", err)
-				if notifyerr := notifyUser("failed to list RAG files", err.Error()); notifyerr != nil {
-					logger.Error("failed to send notification", "error", notifyerr)
-				}
-				return nil
+				loadedFiles = []string{} // Continue with empty list on error
 			}
-			chatLoadedRAGTable := makeLoadedRAGTable(fileList)
-			pages.AddPage(RAGLoadedPage, chatLoadedRAGTable, true, true)
+			chatRAGTable := makeRAGTable(fileList, loadedFiles)
+			pages.AddPage(RAGPage, chatRAGTable, true, true)
 			return nil
 		}
 		if event.Key() == tcell.KeyRune && event.Modifiers() == tcell.ModAlt && event.Rune() == '1' {
@@ -933,55 +1024,63 @@ func init() {
 		}
 		// cannot send msg in editMode or botRespMode
 		if event.Key() == tcell.KeyEscape && !editMode && !botRespMode {
-			msgText := textArea.GetText()
-			if shellMode && msgText != "" {
-				// In shell mode, execute command instead of sending to LLM
-				executeCommandAndDisplay(msgText)
-				textArea.SetText("", true) // Clear the input area
+			if shellMode {
+				cmdText := shellInput.GetText()
+				if cmdText != "" {
+					executeCommandAndDisplay(cmdText)
+					shellInput.SetText("")
+				}
 				return nil
-			} else if !shellMode {
-				// Normal mode - send to LLM
-				nl := "\n\n" // keep empty lines between messages
-				prevText := textView.GetText(true)
-				persona := cfg.UserRole
-				// strings.LastIndex()
-				// newline is not needed is prev msg ends with one
-				if strings.HasSuffix(prevText, nl) {
-					nl = ""
-				} else if strings.HasSuffix(prevText, "\n") {
-					nl = "\n" // only one newline, add another
+			}
+			msgText := textArea.GetText()
+			nl := "\n\n" // keep empty lines between messages
+			prevText := textView.GetText(true)
+			persona := cfg.UserRole
+			// strings.LastIndex()
+			// newline is not needed is prev msg ends with one
+			if strings.HasSuffix(prevText, nl) {
+				nl = ""
+			} else if strings.HasSuffix(prevText, "\n") {
+				nl = "\n" // only one newline, add another
+			}
+			if msgText != "" {
+				// as what char user sends msg?
+				if cfg.WriteNextMsgAs != "" {
+					persona = cfg.WriteNextMsgAs
 				}
-				if msgText != "" {
-					// as what char user sends msg?
-					if cfg.WriteNextMsgAs != "" {
-						persona = cfg.WriteNextMsgAs
+				// check if plain text
+				if !injectRole {
+					matches := roleRE.FindStringSubmatch(msgText)
+					if len(matches) > 1 {
+						persona = matches[1]
+						msgText = strings.TrimLeft(msgText[len(matches[0]):], " ")
 					}
-					// check if plain text
-					if !injectRole {
-						matches := roleRE.FindStringSubmatch(msgText)
-						if len(matches) > 1 {
-							persona = matches[1]
-							msgText = strings.TrimLeft(msgText[len(matches[0]):], " ")
-						}
-					}
-					// add user icon before user msg
-					fmt.Fprintf(textView, "%s[-:-:b](%d) <%s>: [-:-:-]\n%s\n",
-						nl, len(chatBody.Messages), persona, msgText)
-					textArea.SetText("", true)
-					if scrollToEndEnabled {
-						textView.ScrollToEnd()
-					}
-					colorText()
 				}
-				// go chatRound(msgText, persona, textView, false, false)
-				chatRoundChan <- &models.ChatRoundReq{Role: persona, UserMsg: msgText}
-				// Also clear any image attachment after sending the message
-				go func() {
-					// Wait a short moment for the message to be processed, then clear the image attachment
-					// This allows the image to be sent with the current message if it was attached
-					// But clears it for the next message
-					ClearImageAttachment()
-				}()
+				// add user icon before user msg
+				fmt.Fprintf(textView, "%s[-:-:b](%d) <%s>: [-:-:-]\n%s\n",
+					nl, len(chatBody.Messages), persona, msgText)
+				textArea.SetText("", true)
+				if scrollToEndEnabled {
+					textView.ScrollToEnd()
+				}
+				colorText()
+			} else {
+				pages.AddPage(confirmPageName, confirmModal, true, true)
+				return nil
+			}
+			// go chatRound(msgText, persona, textView, false, false)
+			chatRoundChan <- &models.ChatRoundReq{Role: persona, UserMsg: msgText}
+			return nil
+		}
+		if event.Key() == tcell.KeyTab {
+			currentF := app.GetFocus()
+			if currentF == textArea {
+				currentText := textArea.GetText()
+				atIndex := strings.LastIndex(currentText, "@")
+				if atIndex >= 0 {
+					filter := currentText[atIndex+1:]
+					showTextAreaFileCompletionPopup(filter)
+				}
 			}
 			return nil
 		}