Enha: avoid \n\n in tool collapse

logic requires reference to config

.gitignore

@@ -5,7 +5,8 @@ history/
 *.db
 config.toml
 sysprompts/*
-!sysprompts/cluedo.json
+!sysprompts/alice_bob_carl.json
+!sysprompts/coding_assistant.json
 history_bak/
 .aider*
 tags

.golangci.yml

@@ -1,6 +1,7 @@
 version: "2"
 run:
-  concurrency: 2
+  timeout: 1m
+  concurrency: 4
   tests: false
 linters:
   default: none
@@ -14,7 +15,13 @@ linters:
     - prealloc
     - staticcheck
     - unused
+    - gocritic
+    - unconvert
+    - wastedassign
   settings:
+    gocritic:
+      enabled-tags:
+        - performance
     funlen:
       lines: 80
       statements: 50

Makefile

@@ -1,23 +1,35 @@
-.PHONY: setconfig run lint setup-whisper build-whisper download-whisper-model docker-up docker-down docker-logs noextra-run noextra-server
+.PHONY: setconfig run lint lintall install-linters setup-whisper build-whisper download-whisper-model docker-up docker-down docker-logs noextra-run installdelve checkdelve
 
 run: setconfig
 	go build -tags extra -o gf-lt && ./gf-lt
 
-server: setconfig
-	go build -tags extra -o gf-lt && ./gf-lt -port 3333
+build-debug:
+	go build -gcflags="all=-N -l" -tags extra -o gf-lt
+
+debug: build-debug
+	dlv exec --headless --accept-multiclient --listen=:2345 ./gf-lt
 
 noextra-run: setconfig
 	go build -tags '!extra' -o gf-lt && ./gf-lt
 
-noextra-server: setconfig
-	go build -tags '!extra' -o gf-lt && ./gf-lt -port 3333
-
 setconfig:
 	find config.toml &>/dev/null || cp config.example.toml config.toml
 
+installdelve:
+	go install github.com/go-delve/delve/cmd/dlv@latest
+
+checkdelve:
+	which dlv &>/dev/null || installdelve
+
+install-linters: ## Install additional linters (noblanks)
+	go install github.com/GrailFinder/noblanks-linter/cmd/noblanks@latest
+
 lint: ## Run linters. Use make install-linters first.
 	golangci-lint run -c .golangci.yml ./...
 
+lintall: lint
+	noblanks ./...
+
 # Whisper STT Setup (in batteries directory)
 setup-whisper: build-whisper download-whisper-model
 

README.md

@@ -8,6 +8,7 @@ made with use of [tview](https://github.com/rivo/tview)
 - tts/stt (run make commands to get deps);
 - image input;
 - function calls (function calls are implemented natively, to avoid calling outside sources);
+- [character specific context (unique feature)](docs/char-specific-context.md)
 
 #### how it looks
 ![how it looks](assets/ex01.png)

agent/request.go

@@ -15,10 +15,10 @@ import (
 var httpClient = &http.Client{}
 
 var defaultProps = map[string]float32{
-	"temperature":     0.8,
-	"dry_multiplier":  0.0,
-	"min_p":           0.05,
-	"n_predict":       -1.0,
+	"temperature":    0.8,
+	"dry_multiplier": 0.0,
+	"min_p":          0.05,
+	"n_predict":      -1.0,
 }
 
 func detectAPI(api string) (isCompletion, isChat, isDeepSeek, isOpenRouter bool) {
@@ -71,23 +71,24 @@ func (ag *AgentClient) buildRequest(sysprompt, msg string) ([]byte, error) {
 	// Build prompt for completion endpoints
 	if isCompletion {
 		var sb strings.Builder
-		for _, m := range messages {
-			sb.WriteString(m.ToPrompt())
+		for i := range messages {
+			sb.WriteString(messages[i].ToPrompt())
 			sb.WriteString("\n")
 		}
 		prompt := strings.TrimSpace(sb.String())
 
-		if isDeepSeek {
+		switch {
+		case isDeepSeek:
 			// DeepSeek completion
 			req := models.NewDSCompletionReq(prompt, model, defaultProps["temperature"], []string{})
 			req.Stream = false // Agents don't need streaming
 			return json.Marshal(req)
-		} else if isOpenRouter {
+		case isOpenRouter:
 			// OpenRouter completion
 			req := models.NewOpenRouterCompletionReq(model, prompt, defaultProps, []string{})
 			req.Stream = false // Agents don't need streaming
 			return json.Marshal(req)
-		} else {
+		default:
 			// Assume llama.cpp completion
 			req := models.NewLCPReq(prompt, model, nil, defaultProps, []string{})
 			req.Stream = false // Agents don't need streaming
@@ -103,15 +104,16 @@ func (ag *AgentClient) buildRequest(sysprompt, msg string) ([]byte, error) {
 			Messages: messages,
 		}
 
-		if isDeepSeek {
+		switch {
+		case isDeepSeek:
 			// DeepSeek chat
 			req := models.NewDSChatReq(*chatBody)
 			return json.Marshal(req)
-		} else if isOpenRouter {
-			// OpenRouter chat
-			req := models.NewOpenRouterChatReq(*chatBody, defaultProps)
+		case isOpenRouter:
+			// OpenRouter chat - agents don't use reasoning by default
+			req := models.NewOpenRouterChatReq(*chatBody, defaultProps, "")
 			return json.Marshal(req)
-		} else {
+		default:
 			// Assume llama.cpp chat (OpenAI format)
 			req := models.OpenAIReq{
 				ChatBody: chatBody,
@@ -138,7 +140,6 @@ func (ag *AgentClient) LLMRequest(body io.Reader) ([]byte, error) {
 		ag.log.Error("failed to read request body", "error", err)
 		return nil, err
 	}
-
 	req, err := http.NewRequest("POST", ag.cfg.CurrentAPI, bytes.NewReader(bodyBytes))
 	if err != nil {
 		ag.log.Error("failed to create request", "error", err)
@@ -148,22 +149,18 @@ func (ag *AgentClient) LLMRequest(body io.Reader) ([]byte, error) {
 	req.Header.Add("Content-Type", "application/json")
 	req.Header.Add("Authorization", "Bearer "+ag.getToken())
 	req.Header.Set("Accept-Encoding", "gzip")
-
 	ag.log.Debug("agent LLM request", "url", ag.cfg.CurrentAPI, "body_preview", string(bodyBytes[:min(len(bodyBytes), 500)]))
-
 	resp, err := httpClient.Do(req)
 	if err != nil {
 		ag.log.Error("llamacpp api request failed", "error", err, "url", ag.cfg.CurrentAPI)
 		return nil, err
 	}
 	defer resp.Body.Close()
-
 	responseBytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		ag.log.Error("failed to read response", "error", err)
 		return nil, err
 	}
-
 	if resp.StatusCode >= 400 {
 		ag.log.Error("agent LLM request failed", "status", resp.StatusCode, "response", string(responseBytes[:min(len(responseBytes), 1000)]))
 		return responseBytes, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(responseBytes[:min(len(responseBytes), 200)]))
@@ -176,7 +173,6 @@ func (ag *AgentClient) LLMRequest(body io.Reader) ([]byte, error) {
 		// Return raw response as fallback
 		return responseBytes, nil
 	}
-
 	return []byte(text), nil
 }
 

bot_test.go

@@ -1,12 +1,10 @@
 package main
-
 import (
 	"gf-lt/config"
 	"gf-lt/models"
 	"reflect"
 	"testing"
 )
-
 func TestConsolidateConsecutiveAssistantMessages(t *testing.T) {
 	// Mock config for testing
 	testCfg := &config.Config{
@@ -14,7 +12,6 @@ func TestConsolidateConsecutiveAssistantMessages(t *testing.T) {
 		WriteNextMsgAsCompletionAgent: "",
 	}
 	cfg = testCfg
-
 	tests := []struct {
 		name     string
 		input    []models.RoleMsg
@@ -114,38 +111,31 @@ func TestConsolidateConsecutiveAssistantMessages(t *testing.T) {
 			},
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			result := consolidateConsecutiveAssistantMessages(tt.input)
-			
+			result := consolidateAssistantMessages(tt.input)
 			if len(result) != len(tt.expected) {
 				t.Errorf("Expected %d messages, got %d", len(tt.expected), len(result))
 				t.Logf("Result: %+v", result)
 				t.Logf("Expected: %+v", tt.expected)
 				return
 			}
-			
 			for i, expectedMsg := range tt.expected {
 				if i >= len(result) {
 					t.Errorf("Result has fewer messages than expected at index %d", i)
 					continue
 				}
-				
 				actualMsg := result[i]
 				if actualMsg.Role != expectedMsg.Role {
 					t.Errorf("Message %d: expected role '%s', got '%s'", i, expectedMsg.Role, actualMsg.Role)
 				}
-				
 				if actualMsg.Content != expectedMsg.Content {
 					t.Errorf("Message %d: expected content '%s', got '%s'", i, expectedMsg.Content, actualMsg.Content)
 				}
-				
 				if actualMsg.ToolCallID != expectedMsg.ToolCallID {
 					t.Errorf("Message %d: expected ToolCallID '%s', got '%s'", i, expectedMsg.ToolCallID, actualMsg.ToolCallID)
 				}
 			}
-			
 			// Additional check: ensure no messages were lost
 			if !reflect.DeepEqual(result, tt.expected) {
 				t.Errorf("Result does not match expected:\nResult:   %+v\nExpected: %+v", result, tt.expected)
@@ -153,7 +143,6 @@ func TestConsolidateConsecutiveAssistantMessages(t *testing.T) {
 		})
 	}
 }
-
 func TestUnmarshalFuncCall(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -213,7 +202,6 @@ func TestUnmarshalFuncCall(t *testing.T) {
 			wantErr: true,
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := unmarshalFuncCall(tt.jsonStr)
@@ -238,7 +226,6 @@ func TestUnmarshalFuncCall(t *testing.T) {
 		})
 	}
 }
-
 func TestConvertJSONToMapStringString(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -265,7 +252,6 @@ func TestConvertJSONToMapStringString(t *testing.T) {
 			wantErr: true,
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := convertJSONToMapStringString(tt.jsonStr)
@@ -287,3 +273,366 @@ func TestConvertJSONToMapStringString(t *testing.T) {
 		})
 	}
 }
+func TestParseKnownToTag(t *testing.T) {
+	tests := []struct {
+		name        string
+		content     string
+		enabled     bool
+		tag         string
+		wantCleaned string
+		wantKnownTo []string
+	}{
+		{
+			name:        "feature disabled returns original",
+			content:     "Hello @Alice@",
+			enabled:     false,
+			tag:         "@",
+			wantCleaned: "Hello @Alice@",
+			wantKnownTo: nil,
+		},
+		{
+			name:        "no tag returns original",
+			content:     "Hello Alice",
+			enabled:     true,
+			tag:         "@",
+			wantCleaned: "Hello Alice",
+			wantKnownTo: nil,
+		},
+		{
+			name:        "single tag with one char",
+			content:     "Hello @Alice@",
+			enabled:     true,
+			tag:         "@",
+			wantCleaned: "Hello",
+			wantKnownTo: []string{"Alice"},
+		},
+		{
+			name:        "single tag with two chars",
+			content:     "Secret @Alice,Bob@ message",
+			enabled:     true,
+			tag:         "@",
+			wantCleaned: "Secret  message",
+			wantKnownTo: []string{"Alice", "Bob"},
+		},
+		{
+			name:        "tag at beginning",
+			content:     "@Alice@ Hello",
+			enabled:     true,
+			tag:         "@",
+			wantCleaned: "Hello",
+			wantKnownTo: []string{"Alice"},
+		},
+		{
+			name:        "tag at end",
+			content:     "Hello @Alice@",
+			enabled:     true,
+			tag:         "@",
+			wantCleaned: "Hello",
+			wantKnownTo: []string{"Alice"},
+		},
+		{
+			name:        "multiple tags",
+			content:     "First @Alice@ then @Bob@",
+			enabled:     true,
+			tag:         "@",
+			wantCleaned: "First  then",
+			wantKnownTo: []string{"Alice", "Bob"},
+		},
+		{
+			name:        "custom tag",
+			content:     "Secret @Alice,Bob@ message",
+			enabled:     true,
+			tag:         "@",
+			wantCleaned: "Secret  message",
+			wantKnownTo: []string{"Alice", "Bob"},
+		},
+		{
+			name:        "empty list",
+			content:     "Secret @@@",
+			enabled:     true,
+			tag:         "@",
+			wantCleaned: "Secret",
+			wantKnownTo: nil,
+		},
+		{
+			name:        "whitespace around commas",
+			content:     "@ Alice , Bob , Carl @",
+			enabled:     true,
+			tag:         "@",
+			wantCleaned: "",
+			wantKnownTo: []string{"Alice", "Bob", "Carl"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Set up config
+			testCfg := &config.Config{
+				CharSpecificContextEnabled: tt.enabled,
+				CharSpecificContextTag:     tt.tag,
+			}
+			cfg = testCfg
+			knownTo := parseKnownToTag(tt.content)
+			if len(knownTo) != len(tt.wantKnownTo) {
+				t.Errorf("parseKnownToTag() knownTo length = %v, want %v", len(knownTo), len(tt.wantKnownTo))
+				t.Logf("got: %v", knownTo)
+				t.Logf("want: %v", tt.wantKnownTo)
+			} else {
+				for i, got := range knownTo {
+					if got != tt.wantKnownTo[i] {
+						t.Errorf("parseKnownToTag() knownTo[%d] = %q, want %q", i, got, tt.wantKnownTo[i])
+					}
+				}
+			}
+		})
+	}
+}
+func TestProcessMessageTag(t *testing.T) {
+	tests := []struct {
+		name    string
+		msg     models.RoleMsg
+		enabled bool
+		tag     string
+		wantMsg models.RoleMsg
+	}{
+		{
+			name: "feature disabled returns unchanged",
+			msg: models.RoleMsg{
+				Role:    "Alice",
+				Content: "Secret @Bob@",
+			},
+			enabled: false,
+			tag:     "@",
+			wantMsg: models.RoleMsg{
+				Role:    "Alice",
+				Content: "Secret @Bob@",
+				KnownTo: nil,
+			},
+		},
+		{
+			name: "no tag, no knownTo",
+			msg: models.RoleMsg{
+				Role:    "Alice",
+				Content: "Hello everyone",
+			},
+			enabled: true,
+			tag:     "@",
+			wantMsg: models.RoleMsg{
+				Role:    "Alice",
+				Content: "Hello everyone",
+				KnownTo: nil,
+			},
+		},
+		{
+			name: "tag with Bob, adds Alice automatically",
+			msg: models.RoleMsg{
+				Role:    "Alice",
+				Content: "Secret @Bob@",
+			},
+			enabled: true,
+			tag:     "@",
+			wantMsg: models.RoleMsg{
+				Role:    "Alice",
+				Content: "Secret",
+				KnownTo: []string{"Bob", "Alice"},
+			},
+		},
+		{
+			name: "tag already includes sender",
+			msg: models.RoleMsg{
+				Role:    "Alice",
+				Content: "@Alice,Bob@",
+			},
+			enabled: true,
+			tag:     "@",
+			wantMsg: models.RoleMsg{
+				Role:    "Alice",
+				Content: "",
+				KnownTo: []string{"Alice", "Bob"},
+			},
+		},
+		{
+			name: "knownTo already set (from DB), tag still processed",
+			msg: models.RoleMsg{
+				Role:    "Alice",
+				Content: "Secret @Bob@",
+				KnownTo: []string{"Alice"}, // from previous processing
+			},
+			enabled: true,
+			tag:     "@",
+			wantMsg: models.RoleMsg{
+				Role:    "Alice",
+				Content: "Secret",
+				KnownTo: []string{"Bob", "Alice"},
+			},
+		},
+		{
+			name: "example from real use",
+			msg: models.RoleMsg{
+				Role:    "Alice",
+				Content: "I'll start with a simple one! The word is 'banana'. (ooc: @Bob@)",
+				KnownTo: []string{"Alice"}, // from previous processing
+			},
+			enabled: true,
+			tag:     "@",
+			wantMsg: models.RoleMsg{
+				Role:    "Alice",
+				Content: "I'll start with a simple one! The word is 'banana'. (ooc: @Bob@)",
+				KnownTo: []string{"Bob", "Alice"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			testCfg := &config.Config{
+				CharSpecificContextEnabled: tt.enabled,
+				CharSpecificContextTag:     tt.tag,
+			}
+			cfg = testCfg
+			got := processMessageTag(&tt.msg)
+			if len(got.KnownTo) != len(tt.wantMsg.KnownTo) {
+				t.Errorf("processMessageTag() KnownTo length = %v, want %v", len(got.KnownTo), len(tt.wantMsg.KnownTo))
+				t.Logf("got: %v", got.KnownTo)
+				t.Logf("want: %v", tt.wantMsg.KnownTo)
+			} else {
+				// order may differ; check membership
+				for _, want := range tt.wantMsg.KnownTo {
+					found := false
+					for _, gotVal := range got.KnownTo {
+						if gotVal == want {
+							found = true
+							break
+						}
+					}
+					if !found {
+						t.Errorf("processMessageTag() missing KnownTo entry %q, got %v", want, got.KnownTo)
+					}
+				}
+			}
+		})
+	}
+}
+func TestFilterMessagesForCharacter(t *testing.T) {
+	messages := []models.RoleMsg{
+		{Role: "system", Content: "System message", KnownTo: nil}, // visible to all
+		{Role: "Alice", Content: "Hello everyone", KnownTo: nil},  // visible to all
+		{Role: "Alice", Content: "Secret for Bob", KnownTo: []string{"Alice", "Bob"}},
+		{Role: "Bob", Content: "Reply to Alice", KnownTo: []string{"Alice", "Bob"}},
+		{Role: "Alice", Content: "Private to Carl", KnownTo: []string{"Alice", "Carl"}},
+		{Role: "Carl", Content: "Hi all", KnownTo: nil}, // visible to all
+	}
+	tests := []struct {
+		name        string
+		enabled     bool
+		character   string
+		wantIndices []int // indices from original messages that should be included
+	}{
+		{
+			name:        "feature disabled returns all",
+			enabled:     false,
+			character:   "Alice",
+			wantIndices: []int{0, 1, 2, 3, 4, 5},
+		},
+		{
+			name:        "character empty returns all",
+			enabled:     true,
+			character:   "",
+			wantIndices: []int{0, 1, 2, 3, 4, 5},
+		},
+		{
+			name:        "Alice sees all including Carl-private",
+			enabled:     true,
+			character:   "Alice",
+			wantIndices: []int{0, 1, 2, 3, 4, 5},
+		},
+		{
+			name:        "Bob sees Alice-Bob secrets and all public",
+			enabled:     true,
+			character:   "Bob",
+			wantIndices: []int{0, 1, 2, 3, 5},
+		},
+		{
+			name:        "Carl sees Alice-Carl secret and public",
+			enabled:     true,
+			character:   "Carl",
+			wantIndices: []int{0, 1, 4, 5},
+		},
+		{
+			name:        "David sees only public messages",
+			enabled:     true,
+			character:   "David",
+			wantIndices: []int{0, 1, 5},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			testCfg := &config.Config{
+				CharSpecificContextEnabled: tt.enabled,
+				CharSpecificContextTag:     "@",
+			}
+			cfg = testCfg
+			got := filterMessagesForCharacter(messages, tt.character)
+			if len(got) != len(tt.wantIndices) {
+				t.Errorf("filterMessagesForCharacter() returned %d messages, want %d", len(got), len(tt.wantIndices))
+				t.Logf("got: %v", got)
+				return
+			}
+			for i, idx := range tt.wantIndices {
+				if got[i].Content != messages[idx].Content {
+					t.Errorf("filterMessagesForCharacter() message %d content = %q, want %q", i, got[i].Content, messages[idx].Content)
+				}
+			}
+		})
+	}
+}
+func TestRoleMsgCopyPreservesKnownTo(t *testing.T) {
+	// Test that the Copy() method preserves the KnownTo field
+	originalMsg := models.RoleMsg{
+		Role:    "Alice",
+		Content: "Test message",
+		KnownTo: []string{"Bob", "Charlie"},
+	}
+	copiedMsg := originalMsg.Copy()
+	if copiedMsg.Role != originalMsg.Role {
+		t.Errorf("Copy() failed to preserve Role: got %q, want %q", copiedMsg.Role, originalMsg.Role)
+	}
+	if copiedMsg.Content != originalMsg.Content {
+		t.Errorf("Copy() failed to preserve Content: got %q, want %q", copiedMsg.Content, originalMsg.Content)
+	}
+	if !reflect.DeepEqual(copiedMsg.KnownTo, originalMsg.KnownTo) {
+		t.Errorf("Copy() failed to preserve KnownTo: got %v, want %v", copiedMsg.KnownTo, originalMsg.KnownTo)
+	}
+	if copiedMsg.ToolCallID != originalMsg.ToolCallID {
+		t.Errorf("Copy() failed to preserve ToolCallID: got %q, want %q", copiedMsg.ToolCallID, originalMsg.ToolCallID)
+	}
+	if copiedMsg.IsContentParts() != originalMsg.IsContentParts() {
+		t.Errorf("Copy() failed to preserve hasContentParts flag")
+	}
+}
+func TestKnownToFieldPreservationScenario(t *testing.T) {
+	// Test the specific scenario from the log where KnownTo field was getting lost
+	originalMsg := models.RoleMsg{
+		Role:    "Alice",
+		Content: `Alice: "Okay, Bob. The word is... **'Ephemeral'**. (ooc: @Bob@)"`,
+		KnownTo: []string{"Bob"}, // This was detected in the log
+	}
+	t.Logf("Original message - Role: %s, Content: %s, KnownTo: %v",
+		originalMsg.Role, originalMsg.Content, originalMsg.KnownTo)
+	// Simulate what happens when the message gets copied during processing
+	copiedMsg := originalMsg.Copy()
+	t.Logf("Copied message - Role: %s, Content: %s, KnownTo: %v",
+		copiedMsg.Role, copiedMsg.Content, copiedMsg.KnownTo)
+	// Check if KnownTo field survived the copy
+	if len(copiedMsg.KnownTo) == 0 {
+		t.Error("ERROR: KnownTo field was lost during copy!")
+	} else {
+		t.Log("SUCCESS: KnownTo field was preserved during copy!")
+	}
+	// Verify the content is the same
+	if copiedMsg.Content != originalMsg.Content {
+		t.Errorf("Content was changed during copy: got %s, want %s", copiedMsg.Content, originalMsg.Content)
+	}
+	// Verify the KnownTo slice is properly copied
+	if !reflect.DeepEqual(copiedMsg.KnownTo, originalMsg.KnownTo) {
+		t.Errorf("KnownTo was not properly copied: got %v, want %v", copiedMsg.KnownTo, originalMsg.KnownTo)
+	}
+}

colors.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"github.com/gdamore/tcell/v2"
+	"github.com/rivo/tview"
+)
+
+var (
+	colorschemes = map[string]tview.Theme{
+		"default": tview.Theme{
+			PrimitiveBackgroundColor:    tcell.ColorDefault,
+			ContrastBackgroundColor:     tcell.ColorGray,
+			MoreContrastBackgroundColor: tcell.ColorSteelBlue,
+			BorderColor:                 tcell.ColorGray,
+			TitleColor:                  tcell.ColorRed,
+			GraphicsColor:               tcell.ColorBlue,
+			PrimaryTextColor:            tcell.ColorLightGray,
+			SecondaryTextColor:          tcell.ColorYellow,
+			TertiaryTextColor:           tcell.ColorOrange,
+			InverseTextColor:            tcell.ColorPurple,
+			ContrastSecondaryTextColor:  tcell.ColorLime,
+		},
+		"gruvbox": tview.Theme{
+			PrimitiveBackgroundColor:    tcell.NewHexColor(0x282828), // Background: #282828 (dark gray)
+			ContrastBackgroundColor:     tcell.ColorDarkGoldenrod,    // Selected option: warm yellow (#b57614)
+			MoreContrastBackgroundColor: tcell.ColorDarkSlateGray,    // Non-selected options: dark grayish-blue (#32302f)
+			BorderColor:                 tcell.ColorLightGray,        // Light gray (#a89984)
+			TitleColor:                  tcell.ColorRed,              // Red (#fb4934)
+			GraphicsColor:               tcell.ColorDarkCyan,         // Cyan (#689d6a)
+			PrimaryTextColor:            tcell.ColorLightGray,        // Light gray (#d5c4a1)
+			SecondaryTextColor:          tcell.ColorYellow,           // Yellow (#fabd2f)
+			TertiaryTextColor:           tcell.ColorOrange,           // Orange (#fe8019)
+			InverseTextColor:            tcell.ColorWhite,            // White (#f9f5d7) for selected text
+			ContrastSecondaryTextColor:  tcell.ColorLightGreen,       // Light green (#b8bb26)
+		},
+		"solarized": tview.Theme{
+			PrimitiveBackgroundColor:    tcell.NewHexColor(0x002b36), // Background: #002b36 (base03)
+			ContrastBackgroundColor:     tcell.ColorDarkCyan,         // Selected option: cyan (#2aa198)
+			MoreContrastBackgroundColor: tcell.ColorDarkSlateGray,    // Non-selected options: dark blue (#073642)
+			BorderColor:                 tcell.ColorLightBlue,        // Light blue (#839496)
+			TitleColor:                  tcell.ColorRed,              // Red (#dc322f)
+			GraphicsColor:               tcell.ColorBlue,             // Blue (#268bd2)
+			PrimaryTextColor:            tcell.ColorWhite,            // White (#fdf6e3)
+			SecondaryTextColor:          tcell.ColorYellow,           // Yellow (#b58900)
+			TertiaryTextColor:           tcell.ColorOrange,           // Orange (#cb4b16)
+			InverseTextColor:            tcell.ColorWhite,            // White (#eee8d5) for selected text
+			ContrastSecondaryTextColor:  tcell.ColorLightCyan,        // Light cyan (#93a1a1)
+		},
+		"dracula": tview.Theme{
+			PrimitiveBackgroundColor:    tcell.NewHexColor(0x282a36), // Background: #282a36
+			ContrastBackgroundColor:     tcell.ColorDarkMagenta,      // Selected option: magenta (#bd93f9)
+			MoreContrastBackgroundColor: tcell.ColorDarkGray,         // Non-selected options: dark gray (#44475a)
+			BorderColor:                 tcell.ColorLightGray,        // Light gray (#f8f8f2)
+			TitleColor:                  tcell.ColorRed,              // Red (#ff5555)
+			GraphicsColor:               tcell.ColorDarkCyan,         // Cyan (#8be9fd)
+			PrimaryTextColor:            tcell.ColorWhite,            // White (#f8f8f2)
+			SecondaryTextColor:          tcell.ColorYellow,           // Yellow (#f1fa8c)
+			TertiaryTextColor:           tcell.ColorOrange,           // Orange (#ffb86c)
+			InverseTextColor:            tcell.ColorWhite,            // White (#f8f8f2) for selected text
+			ContrastSecondaryTextColor:  tcell.ColorLightGreen,       // Light green (#50fa7b)
+		},
+	}
+)

config.example.toml

@@ -10,7 +10,10 @@ DeepSeekModel = "deepseek-reasoner"
 OpenRouterCompletionAPI = "https://openrouter.ai/api/v1/completions"
 OpenRouterChatAPI = "https://openrouter.ai/api/v1/chat/completions"
 # OpenRouterToken = ""
+# embeddings
 EmbedURL = "http://localhost:8082/v1/embeddings"
+HFToken = ""
+#
 ShowSys = true
 LogFile = "log.txt"
 UserRole = "user"
@@ -19,11 +22,11 @@ AssistantRole = "assistant"
 SysDir = "sysprompts"
 ChunkLimit = 100000
 AutoScrollEnabled = true
-# AutoCleanToolCallsFromCtx = false
+AutoCleanToolCallsFromCtx = false
 # rag settings
+RAGEnabled = false
 RAGBatchSize = 1
 RAGWordLimit = 80
-RAGWorkers = 2
 RAGDir = "ragimport"
 # extra tts
 TTS_ENABLED = false
@@ -39,7 +42,17 @@ WhisperBinaryPath = "./batteries/whisper.cpp/build/bin/whisper-cli" # Path to wh
 WhisperModelPath = "./batteries/whisper.cpp/ggml-large-v3-turbo-q5_0.bin"  # Path to whisper model file (for WHISPER_BINARY mode)
 STT_LANG = "en"  # Language for speech recognition (for WHISPER_BINARY mode)
 STT_SR = 16000  # Sample rate for audio recording
+#
 DBPATH = "gflt.db"
-FilePickerDir = "." # Directory where file picker should start
+FilePickerDir = "." # Directory for file picker start and coding assistant file operations (relative paths resolved against this)
 FilePickerExts = "png,jpg,jpeg,gif,webp" # Comma-separated list of allowed file extensions for file picker
 EnableMouse = false # Enable mouse support in the UI
+# character specific context
+CharSpecificContextEnabled = true
+CharSpecificContextTag = "@"
+AutoTurn = true
+StripThinkingFromAPI = true  # Strip <think> blocks from messages before sending to LLM (keeps them in chat history)
+# OpenRouter reasoning configuration (only applies to OpenRouter chat API)
+# Valid values: xhigh, high, medium, low, minimal, none (empty or none = disabled)
+# Models that support reasoning will include thinking content wrapped in <think> tags
+ReasoningEffort = "medium"

config/config.go

@@ -18,7 +18,8 @@ type Config struct {
 	UserRole                      string `toml:"UserRole"`
 	ToolRole                      string `toml:"ToolRole"`
 	ToolUse                       bool   `toml:"ToolUse"`
-	ThinkUse                      bool   `toml:"ThinkUse"`
+	StripThinkingFromAPI          bool   `toml:"StripThinkingFromAPI"`
+	ReasoningEffort               string `toml:"ReasoningEffort"`
 	AssistantRole                 string `toml:"AssistantRole"`
 	SysDir                        string `toml:"SysDir"`
 	ChunkLimit                    uint32 `toml:"ChunkLimit"`
@@ -26,14 +27,18 @@ type Config struct {
 	WriteNextMsgAs                string
 	WriteNextMsgAsCompletionAgent string
 	SkipLLMResp                   bool
-	AutoCleanToolCallsFromCtx     bool `toml:"AutoCleanToolCallsFromCtx"`
+	AutoCleanToolCallsFromCtx     bool   `toml:"AutoCleanToolCallsFromCtx"`
+	DBPATH                        string `toml:"DBPATH"`
+	FilePickerDir                 string `toml:"FilePickerDir"`
+	FilePickerExts                string `toml:"FilePickerExts"`
+	ImagePreview                  bool   `toml:"ImagePreview"`
+	EnableMouse                   bool   `toml:"EnableMouse"`
 	// embeddings
-	RAGEnabled bool   `toml:"RAGEnabled"`
-	EmbedURL   string `toml:"EmbedURL"`
-	HFToken    string `toml:"HFToken"`
-	RAGDir     string `toml:"RAGDir"`
+	EmbedURL string `toml:"EmbedURL"`
+	HFToken  string `toml:"HFToken"`
 	// rag settings
-	RAGWorkers   uint32 `toml:"RAGWorkers"`
+	RAGEnabled   bool   `toml:"RAGEnabled"`
+	RAGDir       string `toml:"RAGDir"`
 	RAGBatchSize int    `toml:"RAGBatchSize"`
 	RAGWordLimit uint32 `toml:"RAGWordLimit"`
 	// deepseek
@@ -61,10 +66,10 @@ type Config struct {
 	WhisperBinaryPath string `toml:"WhisperBinaryPath"`
 	WhisperModelPath  string `toml:"WhisperModelPath"`
 	STT_LANG          string `toml:"STT_LANG"`
-	DBPATH            string `toml:"DBPATH"`
-	FilePickerDir     string `toml:"FilePickerDir"`
-	FilePickerExts    string `toml:"FilePickerExts"`
-	EnableMouse       bool   `toml:"EnableMouse"`
+	// character spefic contetx
+	CharSpecificContextEnabled bool   `toml:"CharSpecificContextEnabled"`
+	CharSpecificContextTag     string `toml:"CharSpecificContextTag"`
+	AutoTurn                   bool   `toml:"AutoTurn"`
 }
 
 func LoadConfig(fn string) (*Config, error) {
@@ -118,6 +123,9 @@ func LoadConfig(fn string) (*Config, error) {
 	if config.CompletionAPI != "" {
 		config.ApiLinks = append(config.ApiLinks, config.CompletionAPI)
 	}
+	if config.RAGDir == "" {
+		config.RAGDir = "ragimport"
+	}
 	// if any value is empty fill with default
 	return config, nil
 }

docs/char-specific-context.md

@@ -0,0 +1,143 @@
+# Character-Specific Context
+
+**/completion only feature; won't work with /v1/chat**
+
+## Overview
+
+Character-Specific Context is a feature that enables private communication between characters in a multi-character chat. When enabled, messages can be tagged with a special marker indicating which characters should "know" about (see) that message. This allows for secret conversations, private information sharing, and roleplaying scenarios where certain characters are not privy to all communications.
+
+(This feature works by filtering the chat history for each character based on the `KnownTo` field associated with each message. Only messages that are intended for a particular character (or are public) are included in that character's view of the conversation.)
+
+## How It Works
+
+### Tagging Messages
+
+Messages can be tagged with a special string (by default `@`) followed by a comma-separated list of character names. The tag can appear anywhere in the message content. **After csv of characters tag should be closed with `@` (for regexp to know where it ends).**
+
+**Example:**
+```
+Alice: @Bob@ Can you keep a secret?
+```
+
+**To avoid breaking immersion, it is better to place the tag in (ooc:)**
+```
+Alice: (ooc: @Bob@) Can you keep a secret?
+```
+
+This message will be visible only to Alice (the sender) and Bob. The tag is parsed by `parseKnownToTag` and the resulting list of character names is stored in the `KnownTo` field of the message (`RoleMsg`). The sender is automatically added to the `KnownTo` list (if not already present) by `processMessageTag`.
+
+Multiple tags can be used in a single message; all mentioned characters are combined into the `KnownTo` list.
+
+### Filtering Chat History
+
+When it's a character's turn to respond, the function `filterMessagesForCharacter` filters the full message list, returning only those messages where:
+
+- `KnownTo` is empty (message is public), OR
+- `KnownTo` contains the character's name.
+
+System messages (`role == "system"`) are always visible to all characters.
+
+The filtered history is then used to construct the prompt sent to the LLM. This ensures each character only sees messages they are supposed to know about.
+
+### Configuration
+
+Two configuration settings control this feature:
+
+- `CharSpecificContextEnabled` – boolean; enables or disables the feature globally.
+- `CharSpecificContextTag` – string; the tag used to mark private messages. Default is `@`.
+
+These are set in `config.toml` (see `config.example.toml` for the default values).
+
+### Processing Pipeline
+
+1. **Message Creation** – When a message is added to the chat (by a user or LLM), `processMessageTag` scans its content for the known‑to tag.
+2. **Storage** – The parsed `KnownTo` list is stored with the message in the database.
+3. **Filtering** – Whenever the chat history is needed (e.g., for an LLM request), `filterMessagesForCharacter` is called with the target character (the one whose turn it is). The filtered list is used for the prompt.
+4. **Display** – The TUI also uses the same filtering when showing the conversation for a selected character (see “Writing as…”).
+
+## Usage Examples
+
+### Basic Private Message
+
+Alice wants to tell Bob something without Carl knowing:
+
+```
+Alice: @Bob@ Meet me at the library tonight.
+```
+
+Result:
+- Alice (sender) sees the message.
+- Bob sees the message.
+- Carl does **not** see the message in his chat history.
+
+### Multi‑recipient Secret
+
+Alice shares a secret with Bob and Carl, but not David:
+
+```
+Alice: (ooc: @Bob,Carl@) The treasure is hidden under the old oak.
+```
+
+### Public Message
+
+A message without any tag (or with an empty `KnownTo`) is visible to all characters.
+
+```
+Alice: Hello everyone!
+```
+
+### User‑Role Considerations
+
+The human user can assume any character’s identity via the “Writing as…” feature (`cfg.UserRole` and `cfg.WriteNextMsgAs`). When the user writes as a character, the same filtering rules apply: the user will see only the messages that character would see.
+
+## Interaction with AutoTurn and WriteNextMsgAsCompletionAgent
+
+### WriteNextMsgAsCompletionAgent
+
+This configuration variable determines which character the LLM should respond as. It is used by `filterMessagesForCurrentCharacter` to select the target character for filtering. If `WriteNextMsgAsCompletionAgent` is set, the LLM will reply in the voice of that character, and only messages visible to that character will be included in the prompt.
+
+### AutoTurn
+
+Normally llm and user (human) take turns writting messages. With private messages there is an issue, where llm can write a private message that will not be visible for character who user controls, so for a human it would appear that llm did not respond. It is desirable in this case, for llm to answer to itself, larping as target character for that private message.
+
+When `AutoTurn` is enabled, the system can automatically trigger responses from llm as characters who have received a private message. The logic in `triggerPrivateMessageResponses` checks the `KnownTo` list of the last message and, for each recipient that is not the user (or the sender), queues a chat round for that character. This creates a chain of private replies without user intervention.
+
+**Example flow:**
+1. Alice (llm) sends a private message to Bob (llm) (`KnownTo = ["Alice","Bob"]`).
+2. Carl (user) sees nothing.
+3. `AutoTurn` detects this and queues a response from Bob.
+4. Bob replies (potentially also privately).
+5. The conversation continues automatically until public message is made, or Carl (user) was included in `KnownTo`.
+
+
+## Cardmaking with multiple characters
+
+So far only json format supports multiple characters.
+[card example](sysprompts/alice_bob_carl.json)
+
+## Limitations & Caveats
+
+### Endpoint Compatibility
+
+Character‑specific context relies on the `/completion` endpoint (or other completion‑style endpoints) where the LLM is presented with a raw text prompt containing the entire filtered history. It does **not** work with OpenAI‑style `/v1/chat/completions` endpoints, because those endpoints enforce a fixed role set (`user`/`assistant`/`system`) and strip custom role names and metadata.
+
+### TTS
+Although text message might be hidden from user character. If TTS is enabled it will be read until tags are parsed. If message should not be viewed by user, tts will stop.
+
+### Tag Parsing
+
+- The tag is case‑sensitive.
+- Whitespace around character names is trimmed.
+- If the tag appears multiple times, all mentioned characters are combined.
+
+### Database Storage
+
+The `KnownTo` field is stored as a JSON array in the database. Existing messages that were created before enabling the feature will have an empty `KnownTo` and thus be visible to all characters.
+
+## Relevant Configuration
+
+```toml
+CharSpecificContextEnabled = true
+CharSpecificContextTag = "@"
+AutoTurn = false
+```

docs/config.md

@@ -80,9 +80,6 @@ This document explains how to set up and configure the application using the `co
 #### RAGWordLimit (`80`)
 - Maximum number of words in a batch to tokenize and store.
 
-#### RAGWorkers (`2`)
-- Number of concurrent workers for RAG processing.
-
 #### RAGDir (`"ragimport"`)
 - Directory containing documents for RAG processing.
 
@@ -140,14 +137,24 @@ This document explains how to set up and configure the application using the `co
 - Path to the SQLite database file used for storing conversation history and other data.
 
 #### FilePickerDir (`"."`)
-- Directory where the file (image) picker should start when selecting files.
-
-#### FilePickerExts (`"png,jpg,jpeg,gif,webp"`)
-- Comma-separated list of allowed file extensions for the file picker.
+- Directory where the file picker starts and where relative paths in coding assistant file tools (file_read, file_write, etc.) are resolved against. Use absolute paths (starting with `/`) to bypass this.
 
 #### EnableMouse (`false`)
 - Enable or disable mouse support in the UI. When set to `true`, allows clicking buttons and interacting with UI elements using the mouse, but prevents the terminal from handling mouse events normally (such as selecting and copying text). When set to `false`, enables default terminal behavior allowing you to select and copy text, but disables mouse interaction with UI elements.
 
+### Character-Specific Context Settings (/completion only)
+
+[character specific context page for more info](./char-specific-context.md)
+
+#### CharSpecificContextEnabled (`true`)
+- Enable or disable character-specific context functionality.
+
+#### CharSpecificContextTag (`"@"`)
+- The tag prefix used to reference character-specific context in messages.
+
+#### AutoTurn (`true`)
+- Enable or disable automatic turn detection/switching.
+
 ### Additional Features
 
 Those could be switched in program, but also bould be setup in config.
@@ -155,8 +162,11 @@ Those could be switched in program, but also bould be setup in config.
 #### ToolUse
 - Enable or disable explanation of tools to llm, so it could use them.
 
-#### ThinkUse
-- Enable or disable insertion of <think> token at the beggining of llm resp.
+### StripThinkingFromAPI (`true`)
+- Strip thinking blocks from messages before sending to LLM. Keeps them in chat history for local viewing but reduces token usage in API calls.
+
+#### ReasoningEffort (`"medium"`)
+- OpenRouter reasoning configuration (only applies to OpenRouter chat API). Valid values: `xhigh`, `high`, `medium`, `low`, `minimal`, `none`. Empty or `none` disables reasoning.
 
 ## Environment Variables
 

docs/tutorial_rp.md

@@ -67,11 +67,9 @@ In case you're running llama.cpp, here is an example of starting the llama.cpp s
 For roleplay, /completion endpoints are much better, since /chat endpoints swap any character name to either `user` or `assistant`.
 Once you have the desired API endpoint
 (for example: http://localhost:8080/completion),
-there are two ways to pick a model:
-- `Ctrl+L` allows you to iterate through the model list while in the main window.
-- `Ctrl+P` (opens the properties table). Go to the `Select a model` row and press Enter. A list of available models will appear; pick any that you want, then press `x` to exit the properties table.
+- `Ctrl+L` to show a model selection popup;
 
-#### Llama.cpp model preload
+#### Llama.cpp model (pre)load
 
 Llama.cpp supports swapping models. To load the picked ones, press `Alt+9`.
 
@@ -128,9 +126,9 @@ The screen flashes briefly as it calculates. "I am experiencing degraded functio
 ```
 
 Once the character name is in history, we can switch who the LLM will respond as by pressing `Ctrl+X`.
-For now, it should be rotating between HAL9000, `Username`, Seraphina, and system.
-Make the status line mention: `Bot will write as Seraphina (ctrl+x)`
-and press Escape to see her reaction.
+For now, it should give a choice between HAL9000, `Username`, Seraphina, and system.
+After the change the status line should say: `Bot will write as Seraphina (ctrl+x)`
+press Escape for llm to write as Seraphina.
 
 #### Image input
 

extra/tts.go

@@ -13,8 +13,8 @@ import (
 	"log/slog"
 	"net/http"
 	"os"
-	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	google_translate_tts "github.com/GrailFinder/google-translate-tts"
@@ -30,43 +30,8 @@ var (
 	TTSFlushChan = make(chan bool, 1)
 	TTSDoneChan  = make(chan bool, 1)
 	// endsWithPunctuation = regexp.MustCompile(`[;.!?]$`)
-	threeOrMoreDashesRE = regexp.MustCompile(`-{3,}`)
 )
 
-// cleanText removes markdown and special characters that are not suitable for TTS
-func cleanText(text string) string {
-	// Remove markdown-like characters that might interfere with TTS
-	text = strings.ReplaceAll(text, "*", "") // Bold/italic markers
-	text = strings.ReplaceAll(text, "#", "") // Headers
-	text = strings.ReplaceAll(text, "_", "") // Underline/italic markers
-	text = strings.ReplaceAll(text, "~", "") // Strikethrough markers
-	text = strings.ReplaceAll(text, "`", "") // Code markers
-	text = strings.ReplaceAll(text, "[", "") // Link brackets
-	text = strings.ReplaceAll(text, "]", "") // Link brackets
-	text = strings.ReplaceAll(text, "!", "") // Exclamation marks (if not punctuation)
-	// Remove HTML tags using regex
-	htmlTagRegex := regexp.MustCompile(`<[^>]*>`)
-	text = htmlTagRegex.ReplaceAllString(text, "")
-	// Split text into lines to handle table separators
-	lines := strings.Split(text, "\n")
-	var filteredLines []string
-	for _, line := range lines {
-		// Check if the line looks like a table separator (e.g., |----|, |===|, | - - - |)
-		// A table separator typically contains only |, -, =, and spaces
-		isTableSeparator := regexp.MustCompile(`^\s*\|\s*[-=\s]+\|\s*$`).MatchString(strings.TrimSpace(line))
-		if !isTableSeparator {
-			// If it's not a table separator, remove vertical bars but keep the content
-			processedLine := strings.ReplaceAll(line, "|", "")
-			filteredLines = append(filteredLines, processedLine)
-		}
-		// If it is a table separator, skip it (don't add to filteredLines)
-	}
-	text = strings.Join(filteredLines, "\n")
-	text = threeOrMoreDashesRE.ReplaceAllString(text, "")
-	text = strings.TrimSpace(text) // Remove leading/trailing whitespace
-	return text
-}
-
 type Orator interface {
 	Speak(text string) error
 	Stop()
@@ -77,6 +42,7 @@ type Orator interface {
 // impl https://github.com/remsky/Kokoro-FastAPI
 type KokoroOrator struct {
 	logger        *slog.Logger
+	mu            sync.Mutex
 	URL           string
 	Format        models.AudioFormat
 	Stream        bool
@@ -93,6 +59,7 @@ type KokoroOrator struct {
 // Google Translate TTS implementation
 type GoogleTranslateOrator struct {
 	logger        *slog.Logger
+	mu            sync.Mutex
 	speech        *google_translate_tts.Speech
 	currentStream *beep.Ctrl
 	currentDone   chan bool
@@ -109,6 +76,7 @@ func (o *KokoroOrator) stoproutine() {
 		for len(TTSTextChan) > 0 {
 			<-TTSTextChan
 		}
+		o.mu.Lock()
 		o.textBuffer.Reset()
 		if o.currentDone != nil {
 			select {
@@ -118,40 +86,46 @@ func (o *KokoroOrator) stoproutine() {
 			}
 		}
 		o.interrupt = true
+		o.mu.Unlock()
 	}
 }
 
 func (o *KokoroOrator) readroutine() {
 	tokenizer, _ := english.NewSentenceTokenizer(nil)
-	// var sentenceBuf bytes.Buffer
-	// var remainder strings.Builder
 	for {
 		select {
 		case chunk := <-TTSTextChan:
+			o.mu.Lock()
 			o.interrupt = false
-			// sentenceBuf.WriteString(chunk)
-			// text := sentenceBuf.String()
 			_, err := o.textBuffer.WriteString(chunk)
 			if err != nil {
 				o.logger.Warn("failed to write to stringbuilder", "error", err)
+				o.mu.Unlock()
 				continue
 			}
 			text := o.textBuffer.String()
 			sentences := tokenizer.Tokenize(text)
 			o.logger.Debug("adding chunk", "chunk", chunk, "text", text, "sen-len", len(sentences))
-			for i, sentence := range sentences {
-				if i == len(sentences)-1 { // last sentence
-					o.textBuffer.Reset()
-					_, err := o.textBuffer.WriteString(sentence.Text)
-					if err != nil {
-						o.logger.Warn("failed to write to stringbuilder", "error", err)
-						continue
-					}
-					continue // if only one (often incomplete) sentence; wait for next chunk
+			if len(sentences) <= 1 {
+				o.mu.Unlock()
+				continue
+			}
+			completeSentences := sentences[:len(sentences)-1]
+			remaining := sentences[len(sentences)-1].Text
+			o.textBuffer.Reset()
+			o.textBuffer.WriteString(remaining)
+			o.mu.Unlock()
+
+			for _, sentence := range completeSentences {
+				o.mu.Lock()
+				interrupted := o.interrupt
+				o.mu.Unlock()
+				if interrupted {
+					return
 				}
-				cleanedText := cleanText(sentence.Text)
+				cleanedText := models.CleanText(sentence.Text)
 				if cleanedText == "" {
-					continue // Skip empty text after cleaning
+					continue
 				}
 				o.logger.Debug("calling Speak with sentence", "sent", cleanedText)
 				if err := o.Speak(cleanedText); err != nil {
@@ -163,7 +137,9 @@ func (o *KokoroOrator) readroutine() {
 			// lln is done get the whole message out
 			if len(TTSTextChan) > 0 { // otherwise might get stuck
 				for chunk := range TTSTextChan {
+					o.mu.Lock()
 					_, err := o.textBuffer.WriteString(chunk)
+					o.mu.Unlock()
 					if err != nil {
 						o.logger.Warn("failed to write to stringbuilder", "error", err)
 						continue
@@ -174,16 +150,21 @@ func (o *KokoroOrator) readroutine() {
 				}
 			}
 			// flush remaining text
+			o.mu.Lock()
 			remaining := o.textBuffer.String()
-			remaining = cleanText(remaining)
+			remaining = models.CleanText(remaining)
 			o.textBuffer.Reset()
+			o.mu.Unlock()
 			if remaining == "" {
 				continue
 			}
 			o.logger.Debug("calling Speak with remainder", "rem", remaining)
 			sentencesRem := tokenizer.Tokenize(remaining)
 			for _, rs := range sentencesRem { // to avoid dumping large volume of text
-				if o.interrupt {
+				o.mu.Lock()
+				interrupt := o.interrupt
+				o.mu.Unlock()
+				if interrupt {
 					break
 				}
 				if err := o.Speak(rs.Text); err != nil {
@@ -240,6 +221,9 @@ func (o *KokoroOrator) GetLogger() *slog.Logger {
 }
 
 func (o *KokoroOrator) requestSound(text string) (io.ReadCloser, error) {
+	if o.URL == "" {
+		return nil, fmt.Errorf("TTS URL is empty")
+	}
 	payload := map[string]interface{}{
 		"input":           text,
 		"voice":           o.Voice,
@@ -291,14 +275,18 @@ func (o *KokoroOrator) Speak(text string) error {
 		o.logger.Debug("failed to init speaker", "error", err)
 	}
 	done := make(chan bool)
+	o.mu.Lock()
 	o.currentDone = done
-	// Create controllable stream and store reference
 	o.currentStream = &beep.Ctrl{Streamer: beep.Seq(streamer, beep.Callback(func() {
+		o.mu.Lock()
 		close(done)
 		o.currentStream = nil
+		o.currentDone = nil
+		o.mu.Unlock()
 	})), Paused: false}
+	o.mu.Unlock()
 	speaker.Play(o.currentStream)
-	<-o.currentDone
+	<-done
 	return nil
 }
 
@@ -307,6 +295,8 @@ func (o *KokoroOrator) Stop() {
 	o.logger.Debug("attempted to stop orator", "orator", o)
 	speaker.Lock()
 	defer speaker.Unlock()
+	o.mu.Lock()
+	defer o.mu.Unlock()
 	if o.currentStream != nil {
 		// o.currentStream.Paused = true
 		o.currentStream.Streamer = nil
@@ -322,6 +312,7 @@ func (o *GoogleTranslateOrator) stoproutine() {
 		for len(TTSTextChan) > 0 {
 			<-TTSTextChan
 		}
+		o.mu.Lock()
 		o.textBuffer.Reset()
 		if o.currentDone != nil {
 			select {
@@ -331,6 +322,7 @@ func (o *GoogleTranslateOrator) stoproutine() {
 			}
 		}
 		o.interrupt = true
+		o.mu.Unlock()
 	}
 }
 
@@ -339,28 +331,37 @@ func (o *GoogleTranslateOrator) readroutine() {
 	for {
 		select {
 		case chunk := <-TTSTextChan:
+			o.mu.Lock()
 			o.interrupt = false
 			_, err := o.textBuffer.WriteString(chunk)
 			if err != nil {
 				o.logger.Warn("failed to write to stringbuilder", "error", err)
+				o.mu.Unlock()
 				continue
 			}
 			text := o.textBuffer.String()
 			sentences := tokenizer.Tokenize(text)
 			o.logger.Debug("adding chunk", "chunk", chunk, "text", text, "sen-len", len(sentences))
-			for i, sentence := range sentences {
-				if i == len(sentences)-1 { // last sentence
-					o.textBuffer.Reset()
-					_, err := o.textBuffer.WriteString(sentence.Text)
-					if err != nil {
-						o.logger.Warn("failed to write to stringbuilder", "error", err)
-						continue
-					}
-					continue // if only one (often incomplete) sentence; wait for next chunk
+			if len(sentences) <= 1 {
+				o.mu.Unlock()
+				continue
+			}
+			completeSentences := sentences[:len(sentences)-1]
+			remaining := sentences[len(sentences)-1].Text
+			o.textBuffer.Reset()
+			o.textBuffer.WriteString(remaining)
+			o.mu.Unlock()
+
+			for _, sentence := range completeSentences {
+				o.mu.Lock()
+				interrupted := o.interrupt
+				o.mu.Unlock()
+				if interrupted {
+					return
 				}
-				cleanedText := cleanText(sentence.Text)
+				cleanedText := models.CleanText(sentence.Text)
 				if cleanedText == "" {
-					continue // Skip empty text after cleaning
+					continue
 				}
 				o.logger.Debug("calling Speak with sentence", "sent", cleanedText)
 				if err := o.Speak(cleanedText); err != nil {
@@ -372,7 +373,9 @@ func (o *GoogleTranslateOrator) readroutine() {
 			// lln is done get the whole message out
 			if len(TTSTextChan) > 0 { // otherwise might get stuck
 				for chunk := range TTSTextChan {
+					o.mu.Lock()
 					_, err := o.textBuffer.WriteString(chunk)
+					o.mu.Unlock()
 					if err != nil {
 						o.logger.Warn("failed to write to stringbuilder", "error", err)
 						continue
@@ -382,16 +385,21 @@ func (o *GoogleTranslateOrator) readroutine() {
 					}
 				}
 			}
+			o.mu.Lock()
 			remaining := o.textBuffer.String()
-			remaining = cleanText(remaining)
+			remaining = models.CleanText(remaining)
 			o.textBuffer.Reset()
+			o.mu.Unlock()
 			if remaining == "" {
 				continue
 			}
 			o.logger.Debug("calling Speak with remainder", "rem", remaining)
 			sentencesRem := tokenizer.Tokenize(remaining)
 			for _, rs := range sentencesRem { // to avoid dumping large volume of text
-				if o.interrupt {
+				o.mu.Lock()
+				interrupt := o.interrupt
+				o.mu.Unlock()
+				if interrupt {
 					break
 				}
 				if err := o.Speak(rs.Text); err != nil {
@@ -434,14 +442,18 @@ func (o *GoogleTranslateOrator) Speak(text string) error {
 		o.logger.Debug("failed to init speaker", "error", err)
 	}
 	done := make(chan bool)
+	o.mu.Lock()
 	o.currentDone = done
-	// Create controllable stream and store reference
 	o.currentStream = &beep.Ctrl{Streamer: beep.Seq(playbackStreamer, beep.Callback(func() {
+		o.mu.Lock()
 		close(done)
 		o.currentStream = nil
+		o.currentDone = nil
+		o.mu.Unlock()
 	})), Paused: false}
+	o.mu.Unlock()
 	speaker.Play(o.currentStream)
-	<-o.currentDone // wait for playback to complete
+	<-done // wait for playback to complete
 	return nil
 }
 
@@ -449,6 +461,8 @@ func (o *GoogleTranslateOrator) Stop() {
 	o.logger.Debug("attempted to stop google translate orator")
 	speaker.Lock()
 	defer speaker.Unlock()
+	o.mu.Lock()
+	defer o.mu.Unlock()
 	if o.currentStream != nil {
 		o.currentStream.Streamer = nil
 	}

go.mod

@@ -6,17 +6,19 @@ require (
 	github.com/BurntSushi/toml v1.5.0
 	github.com/GrailFinder/google-translate-tts v0.1.3
 	github.com/GrailFinder/searchagent v0.2.0
+	github.com/PuerkitoBio/goquery v1.11.0
 	github.com/gdamore/tcell/v2 v2.13.2
 	github.com/glebarez/go-sqlite v1.22.0
 	github.com/gopxl/beep/v2 v2.1.1
 	github.com/gordonklaus/portaudio v0.0.0-20250206071425-98a94950218b
 	github.com/jmoiron/sqlx v1.4.0
+	github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728
 	github.com/neurosnap/sentences v1.1.2
 	github.com/rivo/tview v0.42.0
+	github.com/yuin/goldmark v1.4.13
 )
 
 require (
-	github.com/PuerkitoBio/goquery v1.11.0 // indirect
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/ebitengine/oto/v3 v3.4.0 // indirect

go.sum

@@ -43,6 +43,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
 github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
+github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+ExRDqGQltzXqN/xypdKP86niVn8=
+github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
@@ -67,6 +69,7 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=

helpfuncs.go

@@ -5,15 +5,56 @@ import (
 	"gf-lt/models"
 	"gf-lt/pngmeta"
 	"image"
+	"net/url"
 	"os"
+	"os/exec"
 	"path"
+	"path/filepath"
+	"slices"
 	"strings"
 	"time"
 	"unicode"
 
-	"math/rand/v2"
+	"github.com/rivo/tview"
 )
 
+// Cached model color - updated by background goroutine
+var cachedModelColor string = "orange"
+
+// startModelColorUpdater starts a background goroutine that periodically updates
+// the cached model color. Only runs HTTP requests for local llama.cpp APIs.
+func startModelColorUpdater() {
+	go func() {
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		// Initial check
+		updateCachedModelColor()
+		for range ticker.C {
+			updateCachedModelColor()
+		}
+	}()
+}
+
+// updateCachedModelColor updates the global cachedModelColor variable
+func updateCachedModelColor() {
+	if !isLocalLlamacpp() {
+		cachedModelColor = "orange"
+		return
+	}
+	// Check if model is loaded
+	loaded, err := isModelLoaded(chatBody.Model)
+	if err != nil {
+		// On error, assume not loaded (red)
+		cachedModelColor = "red"
+		return
+	}
+	if loaded {
+		cachedModelColor = "green"
+	} else {
+		cachedModelColor = "red"
+	}
+}
+
 func isASCII(s string) bool {
 	for i := 0; i < len(s); i++ {
 		if s[i] > unicode.MaxASCII {
@@ -23,6 +64,69 @@ func isASCII(s string) bool {
 	return true
 }
 
+func mapToString[V any](m map[string]V) string {
+	rs := strings.Builder{}
+	for k, v := range m {
+		fmt.Fprintf(&rs, "%v: %v\n", k, v)
+	}
+	return rs.String()
+}
+
+// stripThinkingFromMsg removes thinking blocks from assistant messages.
+// Skips user, tool, and system messages as they may contain thinking examples.
+func stripThinkingFromMsg(msg *models.RoleMsg) *models.RoleMsg {
+	if !cfg.StripThinkingFromAPI {
+		return msg
+	}
+	// Skip user, tool, they might contain thinking and system messages - examples
+	if msg.Role == cfg.UserRole || msg.Role == cfg.ToolRole || msg.Role == "system" {
+		return msg
+	}
+	// Strip thinking from assistant messages
+	msgText := msg.GetText()
+	if thinkRE.MatchString(msgText) {
+		cleanedText := thinkRE.ReplaceAllString(msgText, "")
+		cleanedText = strings.TrimSpace(cleanedText)
+		msg.SetText(cleanedText)
+	}
+	return msg
+}
+
+// refreshChatDisplay updates the chat display based on current character view
+// It filters messages for the character the user is currently "writing as"
+// and updates the textView with the filtered conversation
+func refreshChatDisplay() {
+	// Determine which character's view to show
+	viewingAs := cfg.UserRole
+	if cfg.WriteNextMsgAs != "" {
+		viewingAs = cfg.WriteNextMsgAs
+	}
+	// Filter messages for this character
+	filteredMessages := filterMessagesForCharacter(chatBody.Messages, viewingAs)
+	displayText := chatToText(filteredMessages, cfg.ShowSys)
+	textView.SetText(displayText)
+	colorText()
+	updateStatusLine()
+	if scrollToEndEnabled {
+		textView.ScrollToEnd()
+	}
+}
+
+// stopTTSIfNotForUser: character specific context, not meant fot the human to hear
+func stopTTSIfNotForUser(msg *models.RoleMsg) {
+	if strings.Contains(cfg.CurrentAPI, "/chat") || !cfg.CharSpecificContextEnabled {
+		return
+	}
+	viewingAs := cfg.UserRole
+	if cfg.WriteNextMsgAs != "" {
+		viewingAs = cfg.WriteNextMsgAs
+	}
+	// stop tts if msg is not for user
+	if !slices.Contains(msg.KnownTo, viewingAs) && cfg.TTS_ENABLED {
+		TTSDoneChan <- true
+	}
+}
+
 func colorText() {
 	text := textView.GetText(false)
 	quoteReplacer := strings.NewReplacer(
@@ -69,7 +173,6 @@ func colorText() {
 	for i, cb := range codeBlocks {
 		text = strings.Replace(text, fmt.Sprintf(placeholder, i), cb, 1)
 	}
-	logger.Debug("thinking debug", "blocks", thinkBlocks)
 	for i, tb := range thinkBlocks {
 		text = strings.Replace(text, fmt.Sprintf(placeholderThink, i), tb, 1)
 	}
@@ -77,8 +180,8 @@ func colorText() {
 }
 
 func updateStatusLine() {
-	statusLineWidget.SetText(makeStatusLine())
-	helpView.SetText(fmt.Sprintf(helpText, makeStatusLine()))
+	status := makeStatusLine()
+	statusLineWidget.SetText(status)
 }
 
 func initSysCards() ([]string, error) {
@@ -100,23 +203,26 @@ func initSysCards() ([]string, error) {
 	return labels, nil
 }
 
-func startNewChat() {
+func startNewChat(keepSysP bool) {
 	id, err := store.ChatGetMaxID()
 	if err != nil {
 		logger.Error("failed to get chat id", "error", err)
 	}
-	if ok := charToStart(cfg.AssistantRole); !ok {
+	if ok := charToStart(cfg.AssistantRole, keepSysP); !ok {
 		logger.Warn("no such sys msg", "name", cfg.AssistantRole)
 	}
 	// set chat body
 	chatBody.Messages = chatBody.Messages[:2]
-	textView.SetText(chatToText(cfg.ShowSys))
+	textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
 	newChat := &models.Chat{
 		ID:        id + 1,
 		Name:      fmt.Sprintf("%d_%s", id+1, cfg.AssistantRole),
-		Msgs:      string(defaultStarterBytes),
-		Agent:     cfg.AssistantRole,
 		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
+		// chat is written to db when we get first llm response (or any)
+		// actual chat history (messages) would be parsed then
+		Msgs:  "",
+		Agent: cfg.AssistantRole,
 	}
 	activeChatName = newChat.Name
 	chatMap[newChat.Name] = newChat
@@ -166,7 +272,7 @@ func setLogLevel(sl string) {
 }
 
 func listRolesWithUser() []string {
-	roles := chatBody.ListRoles()
+	roles := listChatRoles()
 	// Remove user role if it exists in the list (to avoid duplicates and ensure it's at position 0)
 	filteredRoles := make([]string, 0, len(roles))
 	for _, role := range roles {
@@ -176,6 +282,7 @@ func listRolesWithUser() []string {
 	}
 	// Prepend user role to the beginning of the list
 	result := append([]string{cfg.UserRole}, filteredRoles...)
+	slices.Sort(result)
 	return result
 }
 
@@ -208,6 +315,23 @@ func strInSlice(s string, sl []string) bool {
 	return false
 }
 
+// isLocalLlamacpp checks if the current API is a local llama.cpp instance.
+func isLocalLlamacpp() bool {
+	u, err := url.Parse(cfg.CurrentAPI)
+	if err != nil {
+		return false
+	}
+	host := u.Hostname()
+	return host == "localhost" || host == "127.0.0.1" || host == "::1"
+}
+
+// getModelColor returns the cached color tag for the model name.
+// The cached value is updated by a background goroutine every 5 seconds.
+// For non-local models, returns orange. For local llama.cpp models, returns green if loaded, red if not.
+func getModelColor() string {
+	return cachedModelColor
+}
+
 func makeStatusLine() string {
 	isRecording := false
 	if asr != nil {
@@ -237,18 +361,537 @@ func makeStatusLine() string {
 	} else {
 		shellModeInfo = ""
 	}
-	statusLine := fmt.Sprintf(indexLineCompletion, botRespMode, activeChatName,
-		cfg.ToolUse, chatBody.Model, cfg.SkipLLMResp, cfg.CurrentAPI,
-		isRecording, persona, botPersona, injectRole)
+	// Get model color based on load status for local llama.cpp models
+	modelColor := getModelColor()
+	statusLine := fmt.Sprintf(statusLineTempl, activeChatName,
+		boolColors[cfg.ToolUse], modelColor, chatBody.Model, boolColors[cfg.SkipLLMResp],
+		cfg.CurrentAPI, persona, botPersona)
+	if cfg.STT_ENABLED {
+		recordingS := fmt.Sprintf(" | [%s:-:b]voice recording[-:-:-] (ctrl+r)",
+			boolColors[isRecording])
+		statusLine += recordingS
+	}
+	// completion endpoint
+	if !strings.Contains(cfg.CurrentAPI, "chat") {
+		roleInject := fmt.Sprintf(" | [%s:-:b]role injection[-:-:-] (alt+7)", boolColors[injectRole])
+		statusLine += roleInject
+	}
 	return statusLine + imageInfo + shellModeInfo
 }
 
-var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
-
-func randString(n int) string {
-	b := make([]rune, n)
-	for i := range b {
-		b[i] = letters[rand.IntN(len(letters))]
+// set of roles within card definition and mention in chat history
+func listChatRoles() []string {
+	currentChat, ok := chatMap[activeChatName]
+	cbc := chatBody.ListRoles()
+	if !ok {
+		return cbc
 	}
-	return string(b)
+	currentCard, ok := sysMap[currentChat.Agent]
+	if !ok {
+		// case which won't let to switch roles:
+		// started new chat (basic_sys or any other), at the start it yet be saved or have chatbody
+		// if it does not have a card or chars, it'll return an empty slice
+		// log error
+		logger.Warn("failed to find current card in sysMap", "agent", currentChat.Agent, "sysMap", sysMap)
+		return cbc
+	}
+	charset := []string{}
+	for _, name := range currentCard.Characters {
+		if !strInSlice(name, cbc) {
+			charset = append(charset, name)
+		}
+	}
+	charset = append(charset, cbc...)
+	return charset
+}
+
+func deepseekModelValidator() error {
+	if cfg.CurrentAPI == cfg.DeepSeekChatAPI || cfg.CurrentAPI == cfg.DeepSeekCompletionAPI {
+		if chatBody.Model != "deepseek-chat" && chatBody.Model != "deepseek-reasoner" {
+			if err := notifyUser("bad request", "wrong deepseek model name"); err != nil {
+				logger.Warn("failed ot notify user", "error", err)
+				return err
+			}
+			return nil
+		}
+	}
+	return nil
+}
+
+// == shellmode ==
+
+func toggleShellMode() {
+	shellMode = !shellMode
+	setShellMode(shellMode)
+	if shellMode {
+		shellInput.SetLabel(fmt.Sprintf("[%s]$ ", cfg.FilePickerDir))
+	} else {
+		textArea.SetPlaceholder("input is multiline; press <Enter> to start the next line;\npress <Esc> to send the message.")
+	}
+	updateStatusLine()
+}
+
+func updateFlexLayout() {
+	if fullscreenMode {
+		// flex already contains only focused widget; do nothing
+		return
+	}
+	flex.Clear()
+	flex.AddItem(textView, 0, 40, false)
+	if shellMode {
+		flex.AddItem(shellInput, 0, 10, false)
+	} else {
+		flex.AddItem(textArea, 0, 10, false)
+	}
+	if positionVisible {
+		flex.AddItem(statusLineWidget, 0, 2, false)
+	}
+	// Keep focus on currently focused widget
+	focused := app.GetFocus()
+	switch {
+	case focused == textView:
+		app.SetFocus(textView)
+	case shellMode:
+		app.SetFocus(shellInput)
+	default:
+		app.SetFocus(textArea)
+	}
+}
+
+func executeCommandAndDisplay(cmdText string) {
+	cmdText = strings.TrimSpace(cmdText)
+	if cmdText == "" {
+		fmt.Fprintf(textView, "\n[red]Error: No command provided[-:-:-]\n")
+		if scrollToEndEnabled {
+			textView.ScrollToEnd()
+		}
+		colorText()
+		return
+	}
+	workingDir := cfg.FilePickerDir
+	// Handle cd command specially to update working directory
+	if strings.HasPrefix(cmdText, "cd ") {
+		newDir := strings.TrimPrefix(cmdText, "cd ")
+		newDir = strings.TrimSpace(newDir)
+		// Handle cd ~ or cdHOME
+		if strings.HasPrefix(newDir, "~") {
+			home := os.Getenv("HOME")
+			newDir = strings.Replace(newDir, "~", home, 1)
+		}
+		// Check if directory exists
+		if _, err := os.Stat(newDir); err == nil {
+			workingDir = newDir
+			cfg.FilePickerDir = workingDir
+			// Update shell input label with new directory
+			shellInput.SetLabel(fmt.Sprintf("[%s]$ ", cfg.FilePickerDir))
+			outputContent := workingDir
+			// Add the command being executed to the chat
+			fmt.Fprintf(textView, "\n[-:-:b](%d) <%s>: [-:-:-]\n$ %s\n",
+				len(chatBody.Messages), cfg.ToolRole, cmdText)
+			fmt.Fprintf(textView, "%s\n", outputContent)
+			combinedMsg := models.RoleMsg{
+				Role:    cfg.ToolRole,
+				Content: "$ " + cmdText + "\n\n" + outputContent,
+			}
+			chatBody.Messages = append(chatBody.Messages, combinedMsg)
+			if scrollToEndEnabled {
+				textView.ScrollToEnd()
+			}
+			colorText()
+			return
+		} else {
+			outputContent := "cd: " + newDir + ": No such file or directory"
+			fmt.Fprintf(textView, "\n[-:-:b](%d) <%s>: [-:-:-]\n$ %s\n",
+				len(chatBody.Messages), cfg.ToolRole, cmdText)
+			fmt.Fprintf(textView, "[red]%s[-:-:-]\n", outputContent)
+			combinedMsg := models.RoleMsg{
+				Role:    cfg.ToolRole,
+				Content: "$ " + cmdText + "\n\n" + outputContent,
+			}
+			chatBody.Messages = append(chatBody.Messages, combinedMsg)
+			if scrollToEndEnabled {
+				textView.ScrollToEnd()
+			}
+			colorText()
+			return
+		}
+	}
+
+	// Use /bin/sh to support pipes, redirects, etc.
+	cmd := exec.Command("/bin/sh", "-c", cmdText)
+	cmd.Dir = workingDir
+	// Execute the command and get output
+	output, err := cmd.CombinedOutput()
+	// Add the command being executed to the chat
+	fmt.Fprintf(textView, "\n[-:-:b](%d) <%s>: [-:-:-]\n$ %s\n",
+		len(chatBody.Messages), cfg.ToolRole, cmdText)
+	var outputContent string
+	if err != nil {
+		// Include both output and error
+		errorMsg := "Error: " + err.Error()
+		fmt.Fprintf(textView, "[red]%s[-:-:-]\n", errorMsg)
+		if len(output) > 0 {
+			outputStr := string(output)
+			fmt.Fprintf(textView, "[red]%s[-:-:-]\n", outputStr)
+			outputContent = errorMsg + "\n" + outputStr
+		} else {
+			outputContent = errorMsg
+		}
+	} else {
+		// Only output if successful
+		if len(output) > 0 {
+			outputStr := string(output)
+			fmt.Fprintf(textView, "[green]%s[-:-:-]\n", outputStr)
+			outputContent = outputStr
+		} else {
+			successMsg := "Command executed successfully (no output)"
+			fmt.Fprintf(textView, "[green]%s[-:-:-]\n", successMsg)
+			outputContent = successMsg
+		}
+	}
+	// Combine command and output in a single message for chat history
+	combinedContent := "$ " + cmdText + "\n\n" + outputContent
+	combinedMsg := models.RoleMsg{
+		Role:    cfg.ToolRole,
+		Content: combinedContent,
+	}
+	chatBody.Messages = append(chatBody.Messages, combinedMsg)
+	// Scroll to end and update colors
+	if scrollToEndEnabled {
+		textView.ScrollToEnd()
+	}
+	colorText()
+	// Add command to history (avoid duplicates at the end)
+	if len(shellHistory) == 0 || shellHistory[len(shellHistory)-1] != cmdText {
+		shellHistory = append(shellHistory, cmdText)
+	}
+	shellHistoryPos = -1
+}
+
+// == search ==
+
+// Global variables for search state
+var searchResults []int
+var searchResultLengths []int // To store the length of each match in the formatted string
+var searchIndex int
+var searchText string
+var originalTextForSearch string
+
+// performSearch searches for the given term in the textView content and highlights matches
+func performSearch(term string) {
+	searchText = term
+	if searchText == "" {
+		searchResults = nil
+		searchResultLengths = nil
+		originalTextForSearch = ""
+		// Re-render text without highlights
+		textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
+		colorText()
+		return
+	}
+	// Get formatted text and search directly in it to avoid mapping issues
+	formattedText := textView.GetText(true)
+	originalTextForSearch = formattedText
+	searchTermLower := strings.ToLower(searchText)
+	formattedTextLower := strings.ToLower(formattedText)
+	// Find all occurrences of the search term in the formatted text directly
+	formattedSearchResults := []int{}
+	searchStart := 0
+	for {
+		pos := strings.Index(formattedTextLower[searchStart:], searchTermLower)
+		if pos == -1 {
+			break
+		}
+		absolutePos := searchStart + pos
+		formattedSearchResults = append(formattedSearchResults, absolutePos)
+		searchStart = absolutePos + len(searchText)
+	}
+	if len(formattedSearchResults) == 0 {
+		// No matches found
+		searchResults = nil
+		searchResultLengths = nil
+		notification := "Pattern not found: " + term
+		if err := notifyUser("search", notification); err != nil {
+			logger.Error("failed to send notification", "error", err)
+		}
+		return
+	}
+	// Store the formatted text positions and lengths for accurate highlighting
+	searchResults = formattedSearchResults
+	// Create lengths array - all matches have the same length as the search term
+	searchResultLengths = make([]int, len(formattedSearchResults))
+	for i := range searchResultLengths {
+		searchResultLengths[i] = len(searchText)
+	}
+	searchIndex = 0
+	highlightCurrentMatch()
+}
+
+// highlightCurrentMatch highlights the current search match and scrolls to it
+func highlightCurrentMatch() {
+	if len(searchResults) == 0 || searchIndex >= len(searchResults) {
+		return
+	}
+	// Get the stored formatted text
+	formattedText := originalTextForSearch
+	// For tview to properly support highlighting and scrolling, we need to work with its region system
+	// Instead of just applying highlights, we need to add region tags to the text
+	highlightedText := addRegionTags(formattedText, searchResults, searchResultLengths, searchIndex, searchText)
+	// Update the text view with the text that includes region tags
+	textView.SetText(highlightedText)
+	// Highlight the current region and scroll to it
+	// Need to identify which position in the results array corresponds to the current match
+	// The region ID will be search_<position>_<index>
+	currentRegion := fmt.Sprintf("search_%d_%d", searchResults[searchIndex], searchIndex)
+	textView.Highlight(currentRegion).ScrollToHighlight()
+	// Send notification about which match we're at
+	notification := fmt.Sprintf("Match %d of %d", searchIndex+1, len(searchResults))
+	if err := notifyUser("search", notification); err != nil {
+		logger.Error("failed to send notification", "error", err)
+	}
+}
+
+// showSearchBar shows the search input field as an overlay
+func showSearchBar() {
+	// Create a temporary flex to combine search and main content
+	updatedFlex := tview.NewFlex().SetDirection(tview.FlexRow).
+		AddItem(searchField, 3, 0, true). // Search field at top
+		AddItem(flex, 0, 1, false)        // Main flex layout below
+	// Add the search overlay as a page
+	pages.AddPage(searchPageName, updatedFlex, true, true)
+	app.SetFocus(searchField)
+}
+
+// hideSearchBar hides the search input field
+func hideSearchBar() {
+	pages.RemovePage(searchPageName)
+	// Return focus to the text view
+	app.SetFocus(textView)
+	// Clear the search field
+	searchField.SetText("")
+}
+
+// Global variables for index overlay functionality
+var indexPageName = "indexOverlay"
+
+// showIndexBar shows the index input field as an overlay at the top
+func showIndexBar() {
+	// Create a temporary flex to combine index input and main content
+	updatedFlex := tview.NewFlex().SetDirection(tview.FlexRow).
+		AddItem(indexPickWindow, 3, 0, true). // Index field at top
+		AddItem(flex, 0, 1, false)            // Main flex layout below
+
+	// Add the index overlay as a page
+	pages.AddPage(indexPageName, updatedFlex, true, true)
+	app.SetFocus(indexPickWindow)
+}
+
+// hideIndexBar hides the index input field
+func hideIndexBar() {
+	pages.RemovePage(indexPageName)
+	// Return focus to the text view
+	app.SetFocus(textView)
+	// Clear the index field
+	indexPickWindow.SetText("")
+}
+
+// addRegionTags adds region tags to search matches in the text for tview highlighting
+func addRegionTags(text string, positions []int, lengths []int, currentIdx int, searchTerm string) string {
+	if len(positions) == 0 {
+		return text
+	}
+	var result strings.Builder
+	lastEnd := 0
+	for i, pos := range positions {
+		endPos := pos + lengths[i]
+		// Add text before this match
+		if pos > lastEnd {
+			result.WriteString(text[lastEnd:pos])
+		}
+		// The matched text, which may contain its own formatting tags
+		actualText := text[pos:endPos]
+		// Add region tag and highlighting for this match
+		// Use a unique region id that includes the match index to avoid conflicts
+		regionId := fmt.Sprintf("search_%d_%d", pos, i) // position + index to ensure uniqueness
+		var highlightStart, highlightEnd string
+		if i == currentIdx {
+			// Current match - use different highlighting
+			highlightStart = fmt.Sprintf(`["%s"][yellow:blue:b]`, regionId) // Current match with region and special highlight
+			highlightEnd = `[-:-:-][""]`                                    // Reset formatting and close region
+		} else {
+			// Other matches - use regular highlighting
+			highlightStart = fmt.Sprintf(`["%s"][gold:red:u]`, regionId) // Other matches with region and highlight
+			highlightEnd = `[-:-:-][""]`                                 // Reset formatting and close region
+		}
+		result.WriteString(highlightStart)
+		result.WriteString(actualText)
+		result.WriteString(highlightEnd)
+		lastEnd = endPos
+	}
+	// Add the rest of the text after the last processed match
+	if lastEnd < len(text) {
+		result.WriteString(text[lastEnd:])
+	}
+	return result.String()
+}
+
+// searchNext finds the next occurrence of the search term
+func searchNext() {
+	if len(searchResults) == 0 {
+		if err := notifyUser("search", "No search results to navigate"); err != nil {
+			logger.Error("failed to send notification", "error", err)
+		}
+		return
+	}
+	searchIndex = (searchIndex + 1) % len(searchResults)
+	highlightCurrentMatch()
+}
+
+// searchPrev finds the previous occurrence of the search term
+func searchPrev() {
+	if len(searchResults) == 0 {
+		if err := notifyUser("search", "No search results to navigate"); err != nil {
+			logger.Error("failed to send notification", "error", err)
+		}
+		return
+	}
+	if searchIndex == 0 {
+		searchIndex = len(searchResults) - 1
+	} else {
+		searchIndex--
+	}
+	highlightCurrentMatch()
+}
+
+// == tab completion ==
+
+func scanFiles(dir, filter string) []string {
+	const maxDepth = 3
+	const maxFiles = 50
+	var files []string
+	var scanRecursive func(currentDir string, currentDepth int, relPath string)
+	scanRecursive = func(currentDir string, currentDepth int, relPath string) {
+		if len(files) >= maxFiles {
+			return
+		}
+		if currentDepth > maxDepth {
+			return
+		}
+		entries, err := os.ReadDir(currentDir)
+		if err != nil {
+			return
+		}
+		for _, entry := range entries {
+			if len(files) >= maxFiles {
+				return
+			}
+			name := entry.Name()
+			if strings.HasPrefix(name, ".") {
+				continue
+			}
+			fullPath := name
+			if relPath != "" {
+				fullPath = relPath + "/" + name
+			}
+			if entry.IsDir() {
+				// Recursively scan subdirectories
+				scanRecursive(filepath.Join(currentDir, name), currentDepth+1, fullPath)
+				continue
+			}
+			// Check if file matches filter
+			if filter == "" || strings.HasPrefix(strings.ToLower(fullPath), strings.ToLower(filter)) {
+				files = append(files, fullPath)
+			}
+		}
+	}
+	scanRecursive(dir, 0, "")
+	return files
+}
+
+// models logic that is too complex for models package
+func MsgToText(i int, m *models.RoleMsg) string {
+	var contentStr string
+	var imageIndicators []string
+	if !m.HasContentParts {
+		contentStr = m.Content
+	} else {
+		var textParts []string
+		for _, part := range m.ContentParts {
+			switch p := part.(type) {
+			case models.TextContentPart:
+				if p.Type == "text" {
+					textParts = append(textParts, p.Text)
+				}
+			case models.ImageContentPart:
+				displayPath := p.Path
+				if displayPath == "" {
+					displayPath = "image"
+				} else {
+					displayPath = extractDisplayPath(displayPath, cfg.FilePickerDir)
+				}
+				imageIndicators = append(imageIndicators, fmt.Sprintf("[orange::i][image: %s][-:-:-]", displayPath))
+			case map[string]any:
+				if partType, exists := p["type"]; exists {
+					switch partType {
+					case "text":
+						if textVal, textExists := p["text"]; textExists {
+							if textStr, isStr := textVal.(string); isStr {
+								textParts = append(textParts, textStr)
+							}
+						}
+					case "image_url":
+						var displayPath string
+						if pathVal, pathExists := p["path"]; pathExists {
+							if pathStr, isStr := pathVal.(string); isStr && pathStr != "" {
+								displayPath = extractDisplayPath(pathStr, cfg.FilePickerDir)
+							}
+						}
+						if displayPath == "" {
+							displayPath = "image"
+						}
+						imageIndicators = append(imageIndicators, fmt.Sprintf("[orange::i][image: %s][-:-:-]", displayPath))
+					}
+				}
+			}
+		}
+		contentStr = strings.Join(textParts, " ") + " "
+	}
+	contentStr, _ = strings.CutPrefix(contentStr, m.Role+":")
+	icon := fmt.Sprintf("(%d) <%s>: ", i, m.Role)
+	var finalContent strings.Builder
+	if len(imageIndicators) > 0 {
+		for _, indicator := range imageIndicators {
+			finalContent.WriteString(indicator)
+			finalContent.WriteString("\n")
+		}
+	}
+	finalContent.WriteString(contentStr)
+	if m.Stats != nil {
+		fmt.Fprintf(&finalContent, "\n[gray::i][%d tok, %.1fs, %.1f t/s][-:-:-]", m.Stats.Tokens, m.Stats.Duration, m.Stats.TokensPerSec)
+	}
+	textMsg := fmt.Sprintf("[-:-:b]%s[-:-:-]\n%s\n", icon, finalContent.String())
+	return strings.ReplaceAll(textMsg, "\n\n", "\n")
+}
+
+// extractDisplayPath returns a path suitable for display, potentially relative to imageBaseDir
+func extractDisplayPath(p, bp string) string {
+	if p == "" {
+		return ""
+	}
+	// If base directory is set, try to make path relative to it
+	if bp != "" {
+		if rel, err := filepath.Rel(bp, p); err == nil {
+			// Check if relative path doesn't start with ".." (meaning it's within base dir)
+			// If it starts with "..", we might still want to show it as relative
+			// but for now we show full path if it goes outside base dir
+			if !strings.HasPrefix(rel, "..") {
+				p = rel
+			}
+		}
+	}
+	// Truncate long paths to last 60 characters if needed
+	if len(p) > 60 {
+		return "..." + p[len(p)-60:]
+	}
+	return p
 }

llm.go

@@ -11,12 +11,11 @@ import (
 
 var imageAttachmentPath string // Global variable to track image attachment for next message
 var lastImg string             // for ctrl+j
-var RAGMsg = "Retrieved context for user's query:\n"
 
 // containsToolSysMsg checks if the toolSysMsg already exists in the chat body
 func containsToolSysMsg() bool {
-	for _, msg := range chatBody.Messages {
-		if msg.Role == cfg.ToolRole && msg.Content == toolSysMsg {
+	for i := range chatBody.Messages {
+		if chatBody.Messages[i].Role == cfg.ToolRole && chatBody.Messages[i].Content == toolSysMsg {
 			return true
 		}
 	}
@@ -34,10 +33,31 @@ func ClearImageAttachment() {
 	imageAttachmentPath = ""
 }
 
+// filterMessagesForCurrentCharacter filters messages based on char-specific context.
+// Returns filtered messages and the bot persona role (target character).
+func filterMessagesForCurrentCharacter(messages []models.RoleMsg) ([]models.RoleMsg, string) {
+	botPersona := cfg.AssistantRole
+	if cfg.WriteNextMsgAsCompletionAgent != "" {
+		botPersona = cfg.WriteNextMsgAsCompletionAgent
+	}
+	if cfg == nil || !cfg.CharSpecificContextEnabled {
+		return messages, botPersona
+	}
+	// get last message (written by user) and checck if it has a tag
+	lm := messages[len(messages)-1]
+	recipient, ok := getValidKnowToRecipient(&lm)
+	if ok && recipient != "" {
+		botPersona = recipient
+	}
+	filtered := filterMessagesForCharacter(messages, botPersona)
+	return filtered, botPersona
+}
+
 type ChunkParser interface {
 	ParseChunk([]byte) (*models.TextChunk, error)
 	FormMsg(msg, role string, cont bool) (io.Reader, error)
 	GetToken() string
+	GetAPIType() models.APIType
 }
 
 func choseChunkParser() {
@@ -87,6 +107,10 @@ type OpenRouterChat struct {
 	Model string
 }
 
+func (lcp LCPCompletion) GetAPIType() models.APIType {
+	return models.APITypeCompletion
+}
+
 func (lcp LCPCompletion) GetToken() string {
 	return ""
 }
@@ -98,7 +122,8 @@ func (lcp LCPCompletion) FormMsg(msg, role string, resume bool) (io.Reader, erro
 	if localImageAttachmentPath != "" {
 		imageURL, err := models.CreateImageURLFromPath(localImageAttachmentPath)
 		if err != nil {
-			logger.Error("failed to create image URL from path for completion", "error", err, "path", localImageAttachmentPath)
+			logger.Error("failed to create image URL from path for completion",
+				"error", err, "path", localImageAttachmentPath)
 			return nil, err
 		}
 		// Extract base64 part from data URL (e.g., "data:image/jpeg;base64,...")
@@ -113,46 +138,19 @@ func (lcp LCPCompletion) FormMsg(msg, role string, resume bool) (io.Reader, erro
 	}
 	if msg != "" { // otherwise let the bot to continue
 		newMsg := models.RoleMsg{Role: role, Content: msg}
+		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	if !resume {
-		// if rag - add as system message to avoid conflicts with tool usage
-		if cfg.RAGEnabled {
-			um := chatBody.Messages[len(chatBody.Messages)-1].Content
-			logger.Debug("RAG is enabled, preparing RAG context", "user_message", um)
-			ragResp, err := chatRagUse(um)
-			if err != nil {
-				logger.Error("failed to form a rag msg", "error", err)
-				return nil, err
-			}
-			logger.Debug("RAG response received", "response_len", len(ragResp), "response_preview", ragResp[:min(len(ragResp), 100)])
-			// Use system role for RAG context to avoid conflicts with tool usage
-			ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-			chatBody.Messages = append(chatBody.Messages, ragMsg)
-			logger.Debug("RAG message added to chat body", "message_count", len(chatBody.Messages))
-		}
-	}
+	// sending description of the tools and how to use them
 	if cfg.ToolUse && !resume && role == cfg.UserRole && !containsToolSysMsg() {
-		// add to chat body
 		chatBody.Messages = append(chatBody.Messages, models.RoleMsg{Role: cfg.ToolRole, Content: toolSysMsg})
 	}
-	messages := make([]string, len(chatBody.Messages))
-	for i, m := range chatBody.Messages {
-		messages[i] = m.ToPrompt()
+	filteredMessages, botPersona := filterMessagesForCurrentCharacter(chatBody.Messages)
+	messages := make([]string, len(filteredMessages))
+	for i := range filteredMessages {
+		messages[i] = stripThinkingFromMsg(&filteredMessages[i]).ToPrompt()
 	}
 	prompt := strings.Join(messages, "\n")
-	// strings builder?
-	if !resume {
-		botPersona := cfg.AssistantRole
-		if cfg.WriteNextMsgAsCompletionAgent != "" {
-			botPersona = cfg.WriteNextMsgAsCompletionAgent
-		}
-		botMsgStart := "\n" + botPersona + ":\n"
-		prompt += botMsgStart
-	}
-	if cfg.ThinkUse && !cfg.ToolUse {
-		prompt += "<think>"
-	}
 	// Add multimodal media markers to the prompt text when multimodal data is present
 	// This is required by llama.cpp multimodal models so they know where to insert media
 	if len(multimodalData) > 0 {
@@ -164,10 +162,15 @@ func (lcp LCPCompletion) FormMsg(msg, role string, resume bool) (io.Reader, erro
 		}
 		prompt = sb.String()
 	}
-
+	// needs to be after <__media__> if there are images
+	if !resume {
+		botMsgStart := "\n" + botPersona + ":\n"
+		prompt += botMsgStart
+	}
 	logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
 		"msg", msg, "resume", resume, "prompt", prompt, "multimodal_data_count", len(multimodalData))
-	payload := models.NewLCPReq(prompt, chatBody.Model, multimodalData, defaultLCPProps, chatBody.MakeStopSlice())
+	payload := models.NewLCPReq(prompt, chatBody.Model, multimodalData,
+		defaultLCPProps, chatBody.MakeStopSliceExcluding("", listChatRoles()))
 	data, err := json.Marshal(payload)
 	if err != nil {
 		logger.Error("failed to form a msg", "error", err)
@@ -193,7 +196,11 @@ func (lcp LCPCompletion) ParseChunk(data []byte) (*models.TextChunk, error) {
 	return resp, nil
 }
 
-func (op LCPChat) GetToken() string {
+func (lcp LCPChat) GetAPIType() models.APIType {
+	return models.APITypeChat
+}
+
+func (lcp LCPChat) GetToken() string {
 	return ""
 }
 
@@ -209,11 +216,11 @@ func (op LCPChat) ParseChunk(data []byte) (*models.TextChunk, error) {
 		logger.Warn("LCPChat ParseChunk: no choices in response", "data", string(data))
 		return &models.TextChunk{Finished: true}, nil
 	}
-
+	lastChoice := llmchunk.Choices[len(llmchunk.Choices)-1]
 	resp := &models.TextChunk{
-		Chunk: llmchunk.Choices[len(llmchunk.Choices)-1].Delta.Content,
+		Chunk:     lastChoice.Delta.Content,
+		Reasoning: lastChoice.Delta.ReasoningContent,
 	}
-
 	// Check for tool calls in all choices, not just the last one
 	for _, choice := range llmchunk.Choices {
 		if len(choice.Delta.ToolCalls) > 0 {
@@ -228,8 +235,7 @@ func (op LCPChat) ParseChunk(data []byte) (*models.TextChunk, error) {
 			break // Process only the first tool call
 		}
 	}
-
-	if llmchunk.Choices[len(llmchunk.Choices)-1].FinishReason == "stop" {
+	if lastChoice.FinishReason == "stop" {
 		if resp.Chunk != "" {
 			logger.Error("text inside of finish llmchunk", "chunk", llmchunk)
 		}
@@ -262,7 +268,7 @@ func (op LCPChat) FormMsg(msg, role string, resume bool) (io.Reader, error) {
 				// If image processing fails, fall back to simple text message
 				newMsg = models.NewRoleMsg(role, msg)
 			} else {
-				newMsg.AddImagePart(imageURL)
+				newMsg.AddImagePart(imageURL, localImageAttachmentPath)
 			}
 			// Only clear the global image attachment after successfully processing it in this API call
 			imageAttachmentPath = "" // Clear the attachment after use
@@ -270,42 +276,39 @@ func (op LCPChat) FormMsg(msg, role string, resume bool) (io.Reader, error) {
 			// Create a simple text message
 			newMsg = models.NewRoleMsg(role, msg)
 		}
+		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
-		logger.Debug("LCPChat FormMsg: added message to chatBody", "role", newMsg.Role, "content_len", len(newMsg.Content), "message_count_after_add", len(chatBody.Messages))
-	}
-	if !resume {
-		// if rag - add as system message to avoid conflicts with tool usage
-		if cfg.RAGEnabled {
-			um := chatBody.Messages[len(chatBody.Messages)-1].Content
-			logger.Debug("LCPChat: RAG is enabled, preparing RAG context", "user_message", um)
-			ragResp, err := chatRagUse(um)
-			if err != nil {
-				logger.Error("LCPChat: failed to form a rag msg", "error", err)
-				return nil, err
-			}
-			logger.Debug("LCPChat: RAG response received", "response_len", len(ragResp), "response_preview", ragResp[:min(len(ragResp), 100)])
-			// Use system role for RAG context to avoid conflicts with tool usage
-			ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-			chatBody.Messages = append(chatBody.Messages, ragMsg)
-			logger.Debug("LCPChat: RAG message added to chat body", "role", ragMsg.Role, "rag_content_len", len(ragMsg.Content), "message_count_after_rag", len(chatBody.Messages))
-		}
+		logger.Debug("LCPChat FormMsg: added message to chatBody", "role", newMsg.Role,
+			"content_len", len(newMsg.Content), "message_count_after_add", len(chatBody.Messages))
 	}
+	filteredMessages, _ := filterMessagesForCurrentCharacter(chatBody.Messages)
 	// openai /v1/chat does not support custom roles; needs to be user, assistant, system
+	// Add persona suffix to the last user message to indicate who the assistant should reply as
 	bodyCopy := &models.ChatBody{
-		Messages: make([]models.RoleMsg, len(chatBody.Messages)),
+		Messages: make([]models.RoleMsg, len(filteredMessages)),
 		Model:    chatBody.Model,
 		Stream:   chatBody.Stream,
 	}
-	for i, msg := range chatBody.Messages {
-		if msg.Role == cfg.UserRole {
-			bodyCopy.Messages[i] = msg
+	for i := range filteredMessages {
+		strippedMsg := *stripThinkingFromMsg(&filteredMessages[i])
+		switch strippedMsg.Role {
+		case cfg.UserRole:
+			bodyCopy.Messages[i] = strippedMsg
 			bodyCopy.Messages[i].Role = "user"
-		} else {
-			bodyCopy.Messages[i] = msg
+		case cfg.AssistantRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "assistant"
+		case cfg.ToolRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "tool"
+		default:
+			bodyCopy.Messages[i] = strippedMsg
 		}
+		// Clear ToolCalls - they're stored in chat history for display but not sent to LLM
+		// bodyCopy.Messages[i].ToolCall = nil
 	}
 	// Clean null/empty messages to prevent API issues
-	bodyCopy.Messages = cleanNullMessages(bodyCopy.Messages)
+	bodyCopy.Messages = consolidateAssistantMessages(bodyCopy.Messages)
 	req := models.OpenAIReq{
 		ChatBody: bodyCopy,
 		Tools:    nil,
@@ -322,6 +325,10 @@ func (op LCPChat) FormMsg(msg, role string, resume bool) (io.Reader, error) {
 }
 
 // deepseek
+func (ds DeepSeekerCompletion) GetAPIType() models.APIType {
+	return models.APITypeCompletion
+}
+
 func (ds DeepSeekerCompletion) ParseChunk(data []byte) (*models.TextChunk, error) {
 	llmchunk := models.DSCompletionResp{}
 	if err := json.Unmarshal(data, &llmchunk); err != nil {
@@ -346,53 +353,34 @@ func (ds DeepSeekerCompletion) GetToken() string {
 
 func (ds DeepSeekerCompletion) FormMsg(msg, role string, resume bool) (io.Reader, error) {
 	logger.Debug("formmsg deepseekercompletion", "link", cfg.CurrentAPI)
+	if err := deepseekModelValidator(); err != nil {
+		return nil, err
+	}
 	if msg != "" { // otherwise let the bot to continue
 		newMsg := models.RoleMsg{Role: role, Content: msg}
+		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	if !resume {
-		// if rag - add as system message to avoid conflicts with tool usage
-		// TODO: perhaps RAG should be a func/tool call instead?
-		if cfg.RAGEnabled {
-			um := chatBody.Messages[len(chatBody.Messages)-1].Content
-			logger.Debug("DeepSeekerCompletion: RAG is enabled, preparing RAG context", "user_message", um)
-			ragResp, err := chatRagUse(um)
-			if err != nil {
-				logger.Error("DeepSeekerCompletion: failed to form a rag msg", "error", err)
-				return nil, err
-			}
-			logger.Debug("DeepSeekerCompletion: RAG response received", "response_len", len(ragResp), "response_preview", ragResp[:min(len(ragResp), 100)])
-			// Use system role for RAG context to avoid conflicts with tool usage
-			ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-			chatBody.Messages = append(chatBody.Messages, ragMsg)
-			logger.Debug("DeepSeekerCompletion: RAG message added to chat body", "message_count", len(chatBody.Messages))
-		}
-	}
+	// sending description of the tools and how to use them
 	if cfg.ToolUse && !resume && role == cfg.UserRole && !containsToolSysMsg() {
-		// add to chat body
 		chatBody.Messages = append(chatBody.Messages, models.RoleMsg{Role: cfg.ToolRole, Content: toolSysMsg})
 	}
-	messages := make([]string, len(chatBody.Messages))
-	for i, m := range chatBody.Messages {
-		messages[i] = m.ToPrompt()
+	filteredMessages, botPersona := filterMessagesForCurrentCharacter(chatBody.Messages)
+	messages := make([]string, len(filteredMessages))
+	for i := range filteredMessages {
+		messages[i] = stripThinkingFromMsg(&filteredMessages[i]).ToPrompt()
 	}
 	prompt := strings.Join(messages, "\n")
 	// strings builder?
 	if !resume {
-		botPersona := cfg.AssistantRole
-		if cfg.WriteNextMsgAsCompletionAgent != "" {
-			botPersona = cfg.WriteNextMsgAsCompletionAgent
-		}
 		botMsgStart := "\n" + botPersona + ":\n"
 		prompt += botMsgStart
 	}
-	if cfg.ThinkUse && !cfg.ToolUse {
-		prompt += "<think>"
-	}
 	logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
 		"msg", msg, "resume", resume, "prompt", prompt)
 	payload := models.NewDSCompletionReq(prompt, chatBody.Model,
-		defaultLCPProps["temp"], chatBody.MakeStopSlice())
+		defaultLCPProps["temp"],
+		chatBody.MakeStopSliceExcluding("", listChatRoles()))
 	data, err := json.Marshal(payload)
 	if err != nil {
 		logger.Error("failed to form a msg", "error", err)
@@ -401,6 +389,10 @@ func (ds DeepSeekerCompletion) FormMsg(msg, role string, resume bool) (io.Reader
 	return bytes.NewReader(data), nil
 }
 
+func (ds DeepSeekerChat) GetAPIType() models.APIType {
+	return models.APITypeChat
+}
+
 func (ds DeepSeekerChat) ParseChunk(data []byte) (*models.TextChunk, error) {
 	llmchunk := models.DSChatStreamResp{}
 	if err := json.Unmarshal(data, &llmchunk); err != nil {
@@ -430,42 +422,46 @@ func (ds DeepSeekerChat) GetToken() string {
 
 func (ds DeepSeekerChat) FormMsg(msg, role string, resume bool) (io.Reader, error) {
 	logger.Debug("formmsg deepseekerchat", "link", cfg.CurrentAPI)
+	if err := deepseekModelValidator(); err != nil {
+		return nil, err
+	}
 	if msg != "" { // otherwise let the bot continue
 		newMsg := models.RoleMsg{Role: role, Content: msg}
+		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	if !resume {
-		// if rag - add as system message to avoid conflicts with tool usage
-		if cfg.RAGEnabled {
-			um := chatBody.Messages[len(chatBody.Messages)-1].Content
-			logger.Debug("RAG is enabled, preparing RAG context", "user_message", um)
-			ragResp, err := chatRagUse(um)
-			if err != nil {
-				logger.Error("failed to form a rag msg", "error", err)
-				return nil, err
-			}
-			logger.Debug("RAG response received", "response_len", len(ragResp), "response_preview", ragResp[:min(len(ragResp), 100)])
-			// Use system role for RAG context to avoid conflicts with tool usage
-			ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-			chatBody.Messages = append(chatBody.Messages, ragMsg)
-			logger.Debug("RAG message added to chat body", "message_count", len(chatBody.Messages))
-		}
-	}
+	// Create copy of chat body with standardized user role
+	filteredMessages, _ := filterMessagesForCurrentCharacter(chatBody.Messages)
+	// Add persona suffix to the last user message to indicate who the assistant should reply as
 	bodyCopy := &models.ChatBody{
-		Messages: make([]models.RoleMsg, len(chatBody.Messages)),
+		Messages: make([]models.RoleMsg, len(filteredMessages)),
 		Model:    chatBody.Model,
 		Stream:   chatBody.Stream,
 	}
-	for i, msg := range chatBody.Messages {
-		if msg.Role == cfg.UserRole || i == 1 {
-			bodyCopy.Messages[i] = msg
-			bodyCopy.Messages[i].Role = "user"
-		} else {
-			bodyCopy.Messages[i] = msg
+	for i := range filteredMessages {
+		strippedMsg := *stripThinkingFromMsg(&filteredMessages[i])
+		switch strippedMsg.Role {
+		case cfg.UserRole:
+			if i == 1 {
+				bodyCopy.Messages[i] = strippedMsg
+				bodyCopy.Messages[i].Role = "user"
+			} else {
+				bodyCopy.Messages[i] = strippedMsg
+			}
+		case cfg.AssistantRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "assistant"
+		case cfg.ToolRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "tool"
+		default:
+			bodyCopy.Messages[i] = strippedMsg
 		}
+		// Clear ToolCalls - they're stored in chat history for display but not sent to LLM
+		// bodyCopy.Messages[i].ToolCall = nil
 	}
 	// Clean null/empty messages to prevent API issues
-	bodyCopy.Messages = cleanNullMessages(bodyCopy.Messages)
+	bodyCopy.Messages = consolidateAssistantMessages(bodyCopy.Messages)
 	dsBody := models.NewDSChatReq(*bodyCopy)
 	data, err := json.Marshal(dsBody)
 	if err != nil {
@@ -476,6 +472,10 @@ func (ds DeepSeekerChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
 }
 
 // openrouter
+func (or OpenRouterCompletion) GetAPIType() models.APIType {
+	return models.APITypeCompletion
+}
+
 func (or OpenRouterCompletion) ParseChunk(data []byte) (*models.TextChunk, error) {
 	llmchunk := models.OpenRouterCompletionResp{}
 	if err := json.Unmarshal(data, &llmchunk); err != nil {
@@ -502,50 +502,29 @@ func (or OpenRouterCompletion) FormMsg(msg, role string, resume bool) (io.Reader
 	logger.Debug("formmsg openroutercompletion", "link", cfg.CurrentAPI)
 	if msg != "" { // otherwise let the bot to continue
 		newMsg := models.RoleMsg{Role: role, Content: msg}
+		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	if !resume {
-		// if rag - add as system message to avoid conflicts with tool usage
-		if cfg.RAGEnabled {
-			um := chatBody.Messages[len(chatBody.Messages)-1].Content
-			logger.Debug("RAG is enabled, preparing RAG context", "user_message", um)
-			ragResp, err := chatRagUse(um)
-			if err != nil {
-				logger.Error("failed to form a rag msg", "error", err)
-				return nil, err
-			}
-			logger.Debug("RAG response received", "response_len", len(ragResp), "response_preview", ragResp[:min(len(ragResp), 100)])
-			// Use system role for RAG context to avoid conflicts with tool usage
-			ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-			chatBody.Messages = append(chatBody.Messages, ragMsg)
-			logger.Debug("RAG message added to chat body", "message_count", len(chatBody.Messages))
-		}
-	}
+	// sending description of the tools and how to use them
 	if cfg.ToolUse && !resume && role == cfg.UserRole && !containsToolSysMsg() {
-		// add to chat body
 		chatBody.Messages = append(chatBody.Messages, models.RoleMsg{Role: cfg.ToolRole, Content: toolSysMsg})
 	}
-	messages := make([]string, len(chatBody.Messages))
-	for i, m := range chatBody.Messages {
-		messages[i] = m.ToPrompt()
+	filteredMessages, botPersona := filterMessagesForCurrentCharacter(chatBody.Messages)
+	messages := make([]string, len(filteredMessages))
+	for i := range filteredMessages {
+		messages[i] = stripThinkingFromMsg(&filteredMessages[i]).ToPrompt()
 	}
 	prompt := strings.Join(messages, "\n")
 	// strings builder?
 	if !resume {
-		botPersona := cfg.AssistantRole
-		if cfg.WriteNextMsgAsCompletionAgent != "" {
-			botPersona = cfg.WriteNextMsgAsCompletionAgent
-		}
 		botMsgStart := "\n" + botPersona + ":\n"
 		prompt += botMsgStart
 	}
-	if cfg.ThinkUse && !cfg.ToolUse {
-		prompt += "<think>"
-	}
-	ss := chatBody.MakeStopSlice()
+	stopSlice := chatBody.MakeStopSliceExcluding("", listChatRoles())
 	logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
-		"msg", msg, "resume", resume, "prompt", prompt, "stop_strings", ss)
-	payload := models.NewOpenRouterCompletionReq(chatBody.Model, prompt, defaultLCPProps, ss)
+		"msg", msg, "resume", resume, "prompt", prompt, "stop_strings", stopSlice)
+	payload := models.NewOpenRouterCompletionReq(chatBody.Model, prompt,
+		defaultLCPProps, stopSlice)
 	data, err := json.Marshal(payload)
 	if err != nil {
 		logger.Error("failed to form a msg", "error", err)
@@ -555,18 +534,24 @@ func (or OpenRouterCompletion) FormMsg(msg, role string, resume bool) (io.Reader
 }
 
 // chat
+func (or OpenRouterChat) GetAPIType() models.APIType {
+	return models.APITypeChat
+}
+
 func (or OpenRouterChat) ParseChunk(data []byte) (*models.TextChunk, error) {
 	llmchunk := models.OpenRouterChatResp{}
 	if err := json.Unmarshal(data, &llmchunk); err != nil {
 		logger.Error("failed to decode", "error", err, "line", string(data))
 		return nil, err
 	}
+	lastChoice := llmchunk.Choices[len(llmchunk.Choices)-1]
 	resp := &models.TextChunk{
-		Chunk: llmchunk.Choices[len(llmchunk.Choices)-1].Delta.Content,
+		Chunk:     lastChoice.Delta.Content,
+		Reasoning: lastChoice.Delta.Reasoning,
 	}
 	// Handle tool calls similar to LCPChat
-	if len(llmchunk.Choices[len(llmchunk.Choices)-1].Delta.ToolCalls) > 0 {
-		toolCall := llmchunk.Choices[len(llmchunk.Choices)-1].Delta.ToolCalls[0]
+	if len(lastChoice.Delta.ToolCalls) > 0 {
+		toolCall := lastChoice.Delta.ToolCalls[0]
 		resp.ToolChunk = toolCall.Function.Arguments
 		fname := toolCall.Function.Name
 		if fname != "" {
@@ -578,7 +563,7 @@ func (or OpenRouterChat) ParseChunk(data []byte) (*models.TextChunk, error) {
 	if resp.ToolChunk != "" {
 		resp.ToolResp = true
 	}
-	if llmchunk.Choices[len(llmchunk.Choices)-1].FinishReason == "stop" {
+	if lastChoice.FinishReason == "stop" {
 		if resp.Chunk != "" {
 			logger.Error("text inside of finish llmchunk", "chunk", llmchunk)
 		}
@@ -611,7 +596,7 @@ func (or OpenRouterChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
 				// If image processing fails, fall back to simple text message
 				newMsg = models.NewRoleMsg(role, msg)
 			} else {
-				newMsg.AddImagePart(imageURL)
+				newMsg.AddImagePart(imageURL, localImageAttachmentPath)
 			}
 			// Only clear the global image attachment after successfully processing it in this API call
 			imageAttachmentPath = "" // Clear the attachment after use
@@ -619,42 +604,39 @@ func (or OpenRouterChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
 			// Create a simple text message
 			newMsg = models.NewRoleMsg(role, msg)
 		}
+		newMsg = *processMessageTag(&newMsg)
 		chatBody.Messages = append(chatBody.Messages, newMsg)
 	}
-	if !resume {
-		// if rag - add as system message to avoid conflicts with tool usage
-		if cfg.RAGEnabled {
-			um := chatBody.Messages[len(chatBody.Messages)-1].Content
-			logger.Debug("RAG is enabled, preparing RAG context", "user_message", um)
-			ragResp, err := chatRagUse(um)
-			if err != nil {
-				logger.Error("failed to form a rag msg", "error", err)
-				return nil, err
-			}
-			logger.Debug("RAG response received", "response_len", len(ragResp), "response_preview", ragResp[:min(len(ragResp), 100)])
-			// Use system role for RAG context to avoid conflicts with tool usage
-			ragMsg := models.RoleMsg{Role: "system", Content: RAGMsg + ragResp}
-			chatBody.Messages = append(chatBody.Messages, ragMsg)
-			logger.Debug("RAG message added to chat body", "message_count", len(chatBody.Messages))
-		}
-	}
 	// Create copy of chat body with standardized user role
+	filteredMessages, _ := filterMessagesForCurrentCharacter(chatBody.Messages)
+	// Add persona suffix to the last user message to indicate who the assistant should reply as
 	bodyCopy := &models.ChatBody{
-		Messages: make([]models.RoleMsg, len(chatBody.Messages)),
+		Messages: make([]models.RoleMsg, len(filteredMessages)),
 		Model:    chatBody.Model,
 		Stream:   chatBody.Stream,
 	}
-	for i, msg := range chatBody.Messages {
-		bodyCopy.Messages[i] = msg
-		// Standardize role if it's a user role
-		if bodyCopy.Messages[i].Role == cfg.UserRole {
-			bodyCopy.Messages[i] = msg
+	for i := range filteredMessages {
+		strippedMsg := *stripThinkingFromMsg(&filteredMessages[i])
+		switch strippedMsg.Role {
+		case cfg.UserRole:
+			bodyCopy.Messages[i] = strippedMsg
 			bodyCopy.Messages[i].Role = "user"
+		case cfg.AssistantRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "assistant"
+		case cfg.ToolRole:
+			bodyCopy.Messages[i] = strippedMsg
+			bodyCopy.Messages[i].Role = "tool"
+		default:
+			bodyCopy.Messages[i] = strippedMsg
 		}
+		// Clear ToolCalls - they're stored in chat history for display but not sent to LLM
+		// literally deletes data that we need
+		// bodyCopy.Messages[i].ToolCall = nil
 	}
 	// Clean null/empty messages to prevent API issues
-	bodyCopy.Messages = cleanNullMessages(bodyCopy.Messages)
-	orBody := models.NewOpenRouterChatReq(*bodyCopy, defaultLCPProps)
+	bodyCopy.Messages = consolidateAssistantMessages(bodyCopy.Messages)
+	orBody := models.NewOpenRouterChatReq(*bodyCopy, defaultLCPProps, cfg.ReasoningEffort)
 	if cfg.ToolUse && !resume && role != cfg.ToolRole {
 		orBody.Tools = baseTools // set tools to use
 	}

main.go

@@ -1,35 +1,27 @@
 package main
 
 import (
-	"flag"
-	"strconv"
-
 	"github.com/rivo/tview"
 )
 
 var (
-	botRespMode            = false
-	editMode               = false
-	roleEditMode           = false
-	injectRole             = true
-	selectedIndex          = int(-1)
-	currentAPIIndex        = 0 // Index to track current API in ApiLinks slice
-	currentORModelIndex    = 0 // Index to track current OpenRouter model in ORFreeModels slice
-	currentLocalModelIndex = 0 // Index to track current llama.cpp model
-	shellMode              = false
-	// indexLine           = "F12 to show keys help | bot resp mode: [orange:-:b]%v[-:-:-] (F6) | card's char: [orange:-:b]%s[-:-:-] (ctrl+s) | chat: [orange:-:b]%s[-:-:-] (F1) | toolUseAdviced: [orange:-:b]%v[-:-:-] (ctrl+k) | model: [orange:-:b]%s[-:-:-] (ctrl+l) | skip LLM resp: [orange:-:b]%v[-:-:-] (F10)\nAPI_URL: [orange:-:b]%s[-:-:-] (ctrl+v) | ThinkUse: [orange:-:b]%v[-:-:-] (ctrl+p) | Log Level: [orange:-:b]%v[-:-:-] (ctrl+p) | Recording: [orange:-:b]%v[-:-:-] (ctrl+r) | Writing as: [orange:-:b]%s[-:-:-] (ctrl+q)"
-	indexLineCompletion = "F12 to show keys help | bot resp mode: [orange:-:b]%v[-:-:-] (F6) | chat: [orange:-:b]%s[-:-:-] (F1) | toolUseAdviced: [orange:-:b]%v[-:-:-] (ctrl+k) | model: [orange:-:b]%s[-:-:-] (ctrl+l) | skip LLM resp: [orange:-:b]%v[-:-:-] (F10)\nAPI: [orange:-:b]%s[-:-:-] (ctrl+v) | Recording: [orange:-:b]%v[-:-:-] (ctrl+r) | Writing as: [orange:-:b]%s[-:-:-] (ctrl+q) | Bot will write as [orange:-:b]%s[-:-:-] (ctrl+x) | role_inject [orange:-:b]%v[-:-:-]"
-	focusSwitcher       = map[tview.Primitive]tview.Primitive{}
+	boolColors        = map[bool]string{true: "green", false: "red"}
+	botRespMode       = false
+	toolRunningMode   = false
+	editMode          = false
+	roleEditMode      = false
+	injectRole        = true
+	selectedIndex     = int(-1)
+	shellMode         = false
+	shellHistory      []string
+	shellHistoryPos   int = -1
+	thinkingCollapsed     = false
+	toolCollapsed         = true
+	statusLineTempl       = "help (F12) | chat: [orange:-:b]%s[-:-:-] (F1) | [%s:-:b]tool use[-:-:-] (ctrl+k) | model: [%s:-:b]%s[-:-:-] (ctrl+l) | [%s:-:b]skip LLM resp[-:-:-] (F10)\nAPI: [orange:-:b]%s[-:-:-] (ctrl+v) | writing as: [orange:-:b]%s[-:-:-] (ctrl+q) | bot will write as [orange:-:b]%s[-:-:-] (ctrl+x)"
+	focusSwitcher         = map[tview.Primitive]tview.Primitive{}
 )
 
 func main() {
-	apiPort := flag.Int("port", 0, "port to host api")
-	flag.Parse()
-	if apiPort != nil && *apiPort > 3000 {
-		srv := Server{}
-		srv.ListenToRequests(strconv.Itoa(*apiPort))
-		return
-	}
 	pages.AddPage("main", flex, true, true)
 	if err := app.SetRoot(pages,
 		true).EnableMouse(cfg.EnableMouse).EnablePaste(true).Run(); err != nil {

main_test.go

@@ -1,42 +0,0 @@
-package main
-
-import (
-	"gf-lt/models"
-	"fmt"
-	"gf-lt/config"
-	"strings"
-	"testing"
-)
-
-func TestRemoveThinking(t *testing.T) {
-	cases := []struct {
-		cb       *models.ChatBody
-		toolMsgs uint8
-	}{
-		{cb: &models.ChatBody{
-			Stream: true,
-			Messages: []models.RoleMsg{
-				{Role: "tool", Content: "should be ommited"},
-				{Role: "system", Content: "should stay"},
-				{Role: "user", Content: "hello, how are you?"},
-				{Role: "assistant", Content: "Oh, hi. <think>I should thank user and continue the conversation</think> I am geat, thank you! How are you?"},
-			},
-		},
-			toolMsgs: uint8(1),
-		},
-	}
-	for i, tc := range cases {
-					t.Run(fmt.Sprintf("run_%d", i), func(t *testing.T) {
-						cfg = &config.Config{ToolRole: "tool"} // Initialize cfg.ToolRole for test
-						mNum := len(tc.cb.Messages)
-						removeThinking(tc.cb)
-						if len(tc.cb.Messages) != mNum-int(tc.toolMsgs) {
-							t.Errorf("failed to delete tools msg %v; expected %d, got %d", tc.cb.Messages, mNum-int(tc.toolMsgs), len(tc.cb.Messages))
-						}
-						for _, msg := range tc.cb.Messages {
-							if strings.Contains(msg.Content, "<think>") {
-								t.Errorf("msg contains think tag; msg: %s\n", msg.Content)
-							}
-						}
-					})	}
-}

models/card.go

@@ -31,18 +31,20 @@ func (c *CharCardSpec) Simplify(userName, fpath string) *CharCard {
 	fm := strings.ReplaceAll(strings.ReplaceAll(c.FirstMes, "{{char}}", c.Name), "{{user}}", userName)
 	sysPr := strings.ReplaceAll(strings.ReplaceAll(c.Description, "{{char}}", c.Name), "{{user}}", userName)
 	return &CharCard{
-		SysPrompt: sysPr,
-		FirstMsg:  fm,
-		Role:      c.Name,
-		FilePath:  fpath,
+		SysPrompt:  sysPr,
+		FirstMsg:   fm,
+		Role:       c.Name,
+		FilePath:   fpath,
+		Characters: []string{c.Name, userName},
 	}
 }
 
 type CharCard struct {
-	SysPrompt string `json:"sys_prompt"`
-	FirstMsg  string `json:"first_msg"`
-	Role      string `json:"role"`
-	FilePath  string `json:"filepath"`
+	SysPrompt  string   `json:"sys_prompt"`
+	FirstMsg   string   `json:"first_msg"`
+	Role       string   `json:"role"`
+	Characters []string `json:"chars"`
+	FilePath   string   `json:"filepath"`
 }
 
 func (cc *CharCard) ToSpec(userName string) *CharCardSpec {

models/consts.go

@@ -0,0 +1,12 @@
+package models
+
+const (
+	LoadedMark = "(loaded) "
+)
+
+type APIType int
+
+const (
+	APITypeChat APIType = iota
+	APITypeCompletion
+)

models/db.go

@@ -14,7 +14,7 @@ type Chat struct {
 	UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
 }
 
-func (c Chat) ToHistory() ([]RoleMsg, error) {
+func (c *Chat) ToHistory() ([]RoleMsg, error) {
 	resp := []RoleMsg{}
 	if err := json.Unmarshal([]byte(c.Msgs), &resp); err != nil {
 		return nil, err

models/extra.go

@@ -1,8 +1,49 @@
 package models
 
+import (
+	"regexp"
+	"strings"
+)
+
 type AudioFormat string
 
 const (
 	AFWav AudioFormat = "wav"
 	AFMP3 AudioFormat = "mp3"
 )
+
+var threeOrMoreDashesRE = regexp.MustCompile(`-{3,}`)
+
+// CleanText removes markdown and special characters that are not suitable for TTS
+func CleanText(text string) string {
+	// Remove markdown-like characters that might interfere with TTS
+	text = strings.ReplaceAll(text, "*", "") // Bold/italic markers
+	text = strings.ReplaceAll(text, "#", "") // Headers
+	text = strings.ReplaceAll(text, "_", "") // Underline/italic markers
+	text = strings.ReplaceAll(text, "~", "") // Strikethrough markers
+	text = strings.ReplaceAll(text, "`", "") // Code markers
+	text = strings.ReplaceAll(text, "[", "") // Link brackets
+	text = strings.ReplaceAll(text, "]", "") // Link brackets
+	text = strings.ReplaceAll(text, "!", "") // Exclamation marks (if not punctuation)
+	// Remove HTML tags using regex
+	htmlTagRegex := regexp.MustCompile(`<[^>]*>`)
+	text = htmlTagRegex.ReplaceAllString(text, "")
+	// Split text into lines to handle table separators
+	lines := strings.Split(text, "\n")
+	var filteredLines []string
+	for _, line := range lines {
+		// Check if the line looks like a table separator (e.g., |----|, |===|, | - - - |)
+		// A table separator typically contains only |, -, =, and spaces
+		isTableSeparator := regexp.MustCompile(`^\s*\|\s*[-=\s]+\|\s*$`).MatchString(strings.TrimSpace(line))
+		if !isTableSeparator {
+			// If it's not a table separator, remove vertical bars but keep the content
+			processedLine := strings.ReplaceAll(line, "|", "")
+			filteredLines = append(filteredLines, processedLine)
+		}
+		// If it is a table separator, skip it (don't add to filteredLines)
+	}
+	text = strings.Join(filteredLines, "\n")
+	text = threeOrMoreDashesRE.ReplaceAllString(text, "")
+	text = strings.TrimSpace(text) // Remove leading/trailing whitespace
+	return text
+}

models/models.go

@@ -14,6 +14,12 @@ type FuncCall struct {
 	Args map[string]string `json:"args"`
 }
 
+type ToolCall struct {
+	ID   string `json:"id,omitempty"`
+	Name string `json:"name"`
+	Args string `json:"arguments"`
+}
+
 type LLMResp struct {
 	Choices []struct {
 		FinishReason string `json:"finish_reason"`
@@ -51,8 +57,9 @@ type LLMRespChunk struct {
 		FinishReason string `json:"finish_reason"`
 		Index        int    `json:"index"`
 		Delta        struct {
-			Content   string          `json:"content"`
-			ToolCalls []ToolDeltaResp `json:"tool_calls"`
+			Content          string          `json:"content"`
+			ReasoningContent string          `json:"reasoning_content"`
+			ToolCalls        []ToolDeltaResp `json:"tool_calls"`
 		} `json:"delta"`
 	} `json:"choices"`
 	Created int    `json:"created"`
@@ -73,6 +80,7 @@ type TextChunk struct {
 	ToolResp  bool
 	FuncName  string
 	ToolID    string
+	Reasoning string // For models that send reasoning separately (OpenRouter, etc.)
 }
 
 type TextContentPart struct {
@@ -82,6 +90,7 @@ type TextContentPart struct {
 
 type ImageContentPart struct {
 	Type     string `json:"type"`
+	Path     string `json:"path,omitempty"` // Store original file path
 	ImageURL struct {
 		URL string `json:"url"`
 	} `json:"image_url"`
@@ -89,37 +98,59 @@ type ImageContentPart struct {
 
 // RoleMsg represents a message with content that can be either a simple string or structured content parts
 type RoleMsg struct {
-	Role            string        `json:"role"`
-	Content         string        `json:"-"`
-	ContentParts    []interface{} `json:"-"`
-	ToolCallID      string        `json:"tool_call_id,omitempty"` // For tool response messages
-	hasContentParts bool          // Flag to indicate which content type to marshal
+	Role            string         `json:"role"`
+	Content         string         `json:"-"`
+	ContentParts    []any          `json:"-"`
+	ToolCallID      string         `json:"tool_call_id,omitempty"`     // For tool response messages
+	ToolCall        *ToolCall      `json:"tool_call,omitempty"`        // For assistant messages with tool calls
+	IsShellCommand  bool           `json:"is_shell_command,omitempty"` // True for shell command outputs (always shown)
+	KnownTo         []string       `json:"known_to,omitempty"`
+	Stats           *ResponseStats `json:"stats"`
+	HasContentParts bool           // Flag to indicate which content type to marshal
 }
 
 // MarshalJSON implements custom JSON marshaling for RoleMsg
+//
+//nolint:gocritic
 func (m RoleMsg) MarshalJSON() ([]byte, error) {
-	if m.hasContentParts {
+	if m.HasContentParts {
 		// Use structured content format
 		aux := struct {
-			Role       string        `json:"role"`
-			Content    []interface{} `json:"content"`
-			ToolCallID string        `json:"tool_call_id,omitempty"`
+			Role           string         `json:"role"`
+			Content        []any          `json:"content"`
+			ToolCallID     string         `json:"tool_call_id,omitempty"`
+			ToolCall       *ToolCall      `json:"tool_call,omitempty"`
+			IsShellCommand bool           `json:"is_shell_command,omitempty"`
+			KnownTo        []string       `json:"known_to,omitempty"`
+			Stats          *ResponseStats `json:"stats,omitempty"`
 		}{
-			Role:       m.Role,
-			Content:    m.ContentParts,
-			ToolCallID: m.ToolCallID,
+			Role:           m.Role,
+			Content:        m.ContentParts,
+			ToolCallID:     m.ToolCallID,
+			ToolCall:       m.ToolCall,
+			IsShellCommand: m.IsShellCommand,
+			KnownTo:        m.KnownTo,
+			Stats:          m.Stats,
 		}
 		return json.Marshal(aux)
 	} else {
 		// Use simple content format
 		aux := struct {
-			Role       string `json:"role"`
-			Content    string `json:"content"`
-			ToolCallID string `json:"tool_call_id,omitempty"`
+			Role           string         `json:"role"`
+			Content        string         `json:"content"`
+			ToolCallID     string         `json:"tool_call_id,omitempty"`
+			ToolCall       *ToolCall      `json:"tool_call,omitempty"`
+			IsShellCommand bool           `json:"is_shell_command,omitempty"`
+			KnownTo        []string       `json:"known_to,omitempty"`
+			Stats          *ResponseStats `json:"stats,omitempty"`
 		}{
-			Role:       m.Role,
-			Content:    m.Content,
-			ToolCallID: m.ToolCallID,
+			Role:           m.Role,
+			Content:        m.Content,
+			ToolCallID:     m.ToolCallID,
+			ToolCall:       m.ToolCall,
+			IsShellCommand: m.IsShellCommand,
+			KnownTo:        m.KnownTo,
+			Stats:          m.Stats,
 		}
 		return json.Marshal(aux)
 	}
@@ -129,23 +160,35 @@ func (m RoleMsg) MarshalJSON() ([]byte, error) {
 func (m *RoleMsg) UnmarshalJSON(data []byte) error {
 	// First, try to unmarshal as structured content format
 	var structured struct {
-		Role       string        `json:"role"`
-		Content    []interface{} `json:"content"`
-		ToolCallID string        `json:"tool_call_id,omitempty"`
+		Role           string         `json:"role"`
+		Content        []any          `json:"content"`
+		ToolCallID     string         `json:"tool_call_id,omitempty"`
+		ToolCall       *ToolCall      `json:"tool_call,omitempty"`
+		IsShellCommand bool           `json:"is_shell_command,omitempty"`
+		KnownTo        []string       `json:"known_to,omitempty"`
+		Stats          *ResponseStats `json:"stats,omitempty"`
 	}
 	if err := json.Unmarshal(data, &structured); err == nil && len(structured.Content) > 0 {
 		m.Role = structured.Role
 		m.ContentParts = structured.Content
 		m.ToolCallID = structured.ToolCallID
-		m.hasContentParts = true
+		m.ToolCall = structured.ToolCall
+		m.IsShellCommand = structured.IsShellCommand
+		m.KnownTo = structured.KnownTo
+		m.Stats = structured.Stats
+		m.HasContentParts = true
 		return nil
 	}
 
 	// Otherwise, unmarshal as simple content format
 	var simple struct {
-		Role       string `json:"role"`
-		Content    string `json:"content"`
-		ToolCallID string `json:"tool_call_id,omitempty"`
+		Role           string         `json:"role"`
+		Content        string         `json:"content"`
+		ToolCallID     string         `json:"tool_call_id,omitempty"`
+		ToolCall       *ToolCall      `json:"tool_call,omitempty"`
+		IsShellCommand bool           `json:"is_shell_command,omitempty"`
+		KnownTo        []string       `json:"known_to,omitempty"`
+		Stats          *ResponseStats `json:"stats,omitempty"`
 	}
 	if err := json.Unmarshal(data, &simple); err != nil {
 		return err
@@ -153,53 +196,32 @@ func (m *RoleMsg) UnmarshalJSON(data []byte) error {
 	m.Role = simple.Role
 	m.Content = simple.Content
 	m.ToolCallID = simple.ToolCallID
-	m.hasContentParts = false
+	m.ToolCall = simple.ToolCall
+	m.IsShellCommand = simple.IsShellCommand
+	m.KnownTo = simple.KnownTo
+	m.Stats = simple.Stats
+	m.HasContentParts = false
 	return nil
 }
 
-func (m RoleMsg) ToText(i int) string {
-	icon := fmt.Sprintf("(%d)", i)
-
-	// Convert content to string representation
-	contentStr := ""
-	if !m.hasContentParts {
+func (m *RoleMsg) ToPrompt() string {
+	var contentStr string
+	if !m.HasContentParts {
 		contentStr = m.Content
 	} else {
 		// For structured content, just take the text parts
 		var textParts []string
 		for _, part := range m.ContentParts {
-			if partMap, ok := part.(map[string]interface{}); ok {
-				if partType, exists := partMap["type"]; exists && partType == "text" {
-					if textVal, textExists := partMap["text"]; textExists {
-						if textStr, isStr := textVal.(string); isStr {
-							textParts = append(textParts, textStr)
-						}
-					}
+			switch p := part.(type) {
+			case TextContentPart:
+				if p.Type == "text" {
+					textParts = append(textParts, p.Text)
 				}
-			}
-		}
-		contentStr = strings.Join(textParts, " ") + " "
-	}
-
-	// check if already has role annotation (/completion makes them)
-	if !strings.HasPrefix(contentStr, m.Role+":") {
-		icon = fmt.Sprintf("(%d) <%s>: ", i, m.Role)
-	}
-	textMsg := fmt.Sprintf("[-:-:b]%s[-:-:-]\n%s\n", icon, contentStr)
-	return strings.ReplaceAll(textMsg, "\n\n", "\n")
-}
-
-func (m RoleMsg) ToPrompt() string {
-	contentStr := ""
-	if !m.hasContentParts {
-		contentStr = m.Content
-	} else {
-		// For structured content, just take the text parts
-		var textParts []string
-		for _, part := range m.ContentParts {
-			if partMap, ok := part.(map[string]interface{}); ok {
-				if partType, exists := partMap["type"]; exists && partType == "text" {
-					if textVal, textExists := partMap["text"]; textExists {
+			case ImageContentPart:
+				// skip images for text display
+			case map[string]any:
+				if partType, exists := p["type"]; exists && partType == "text" {
+					if textVal, textExists := p["text"]; textExists {
 						if textStr, isStr := textVal.(string); isStr {
 							textParts = append(textParts, textStr)
 						}
@@ -217,81 +239,144 @@ func NewRoleMsg(role, content string) RoleMsg {
 	return RoleMsg{
 		Role:            role,
 		Content:         content,
-		hasContentParts: false,
+		HasContentParts: false,
 	}
 }
 
 // NewMultimodalMsg creates a RoleMsg with structured content parts (text and images)
-func NewMultimodalMsg(role string, contentParts []interface{}) RoleMsg {
+func NewMultimodalMsg(role string, contentParts []any) RoleMsg {
 	return RoleMsg{
 		Role:            role,
 		ContentParts:    contentParts,
-		hasContentParts: true,
+		HasContentParts: true,
 	}
 }
 
 // HasContent returns true if the message has either string content or structured content parts
-func (m RoleMsg) HasContent() bool {
+func (m *RoleMsg) HasContent() bool {
 	if m.Content != "" {
 		return true
 	}
-	if m.hasContentParts && len(m.ContentParts) > 0 {
+	if m.HasContentParts && len(m.ContentParts) > 0 {
 		return true
 	}
 	return false
 }
 
 // IsContentParts returns true if the message uses structured content parts
-func (m RoleMsg) IsContentParts() bool {
-	return m.hasContentParts
+func (m *RoleMsg) IsContentParts() bool {
+	return m.HasContentParts
 }
 
 // GetContentParts returns the content parts of the message
-func (m RoleMsg) GetContentParts() []interface{} {
+func (m *RoleMsg) GetContentParts() []any {
 	return m.ContentParts
 }
 
 // Copy creates a copy of the RoleMsg with all fields
-func (m RoleMsg) Copy() RoleMsg {
+func (m *RoleMsg) Copy() RoleMsg {
 	return RoleMsg{
 		Role:            m.Role,
 		Content:         m.Content,
 		ContentParts:    m.ContentParts,
 		ToolCallID:      m.ToolCallID,
-		hasContentParts: m.hasContentParts,
+		KnownTo:         m.KnownTo,
+		Stats:           m.Stats,
+		HasContentParts: m.HasContentParts,
+		ToolCall:        m.ToolCall,
+		IsShellCommand:  m.IsShellCommand,
 	}
 }
 
+// GetText returns the text content of the message, handling both
+// simple Content and multimodal ContentParts formats.
+func (m *RoleMsg) GetText() string {
+	if !m.HasContentParts {
+		return m.Content
+	}
+	var textParts []string
+	for _, part := range m.ContentParts {
+		switch p := part.(type) {
+		case TextContentPart:
+			if p.Type == "text" {
+				textParts = append(textParts, p.Text)
+			}
+		case map[string]any:
+			if partType, exists := p["type"]; exists {
+				if partType == "text" {
+					if textVal, textExists := p["text"]; textExists {
+						if textStr, isStr := textVal.(string); isStr {
+							textParts = append(textParts, textStr)
+						}
+					}
+				}
+			}
+		}
+	}
+	return strings.Join(textParts, " ")
+}
+
+// SetText updates the text content of the message. If the message has
+// ContentParts (multimodal), it updates the text parts while preserving
+// images. If not, it sets the simple Content field.
+func (m *RoleMsg) SetText(text string) {
+	if !m.HasContentParts {
+		m.Content = text
+		return
+	}
+	var newParts []any
+	for _, part := range m.ContentParts {
+		switch p := part.(type) {
+		case TextContentPart:
+			if p.Type == "text" {
+				p.Text = text
+				newParts = append(newParts, p)
+			} else {
+				newParts = append(newParts, p)
+			}
+		case map[string]any:
+			if partType, exists := p["type"]; exists && partType == "text" {
+				p["text"] = text
+				newParts = append(newParts, p)
+			} else {
+				newParts = append(newParts, p)
+			}
+		default:
+			newParts = append(newParts, part)
+		}
+	}
+	m.ContentParts = newParts
+}
+
 // AddTextPart adds a text content part to the message
 func (m *RoleMsg) AddTextPart(text string) {
-	if !m.hasContentParts {
+	if !m.HasContentParts {
 		// Convert to content parts format
 		if m.Content != "" {
-			m.ContentParts = []interface{}{TextContentPart{Type: "text", Text: m.Content}}
+			m.ContentParts = []any{TextContentPart{Type: "text", Text: m.Content}}
 		} else {
-			m.ContentParts = []interface{}{}
+			m.ContentParts = []any{}
 		}
-		m.hasContentParts = true
+		m.HasContentParts = true
 	}
-
 	textPart := TextContentPart{Type: "text", Text: text}
 	m.ContentParts = append(m.ContentParts, textPart)
 }
 
 // AddImagePart adds an image content part to the message
-func (m *RoleMsg) AddImagePart(imageURL string) {
-	if !m.hasContentParts {
+func (m *RoleMsg) AddImagePart(imageURL, imagePath string) {
+	if !m.HasContentParts {
 		// Convert to content parts format
 		if m.Content != "" {
-			m.ContentParts = []interface{}{TextContentPart{Type: "text", Text: m.Content}}
+			m.ContentParts = []any{TextContentPart{Type: "text", Text: m.Content}}
 		} else {
-			m.ContentParts = []interface{}{}
+			m.ContentParts = []any{}
 		}
-		m.hasContentParts = true
+		m.HasContentParts = true
 	}
-
 	imagePart := ImageContentPart{
 		Type: "image_url",
+		Path: imagePath, // Store the original file path
 		ImageURL: struct {
 			URL string `json:"url"`
 		}{URL: imageURL},
@@ -338,16 +423,16 @@ type ChatBody struct {
 }
 
 func (cb *ChatBody) Rename(oldname, newname string) {
-	for i, m := range cb.Messages {
-		cb.Messages[i].Content = strings.ReplaceAll(m.Content, oldname, newname)
-		cb.Messages[i].Role = strings.ReplaceAll(m.Role, oldname, newname)
+	for i := range cb.Messages {
+		cb.Messages[i].Content = strings.ReplaceAll(cb.Messages[i].Content, oldname, newname)
+		cb.Messages[i].Role = strings.ReplaceAll(cb.Messages[i].Role, oldname, newname)
 	}
 }
 
 func (cb *ChatBody) ListRoles() []string {
 	namesMap := make(map[string]struct{})
-	for _, m := range cb.Messages {
-		namesMap[m.Role] = struct{}{}
+	for i := range cb.Messages {
+		namesMap[cb.Messages[i].Role] = struct{}{}
 	}
 	resp := make([]string, len(namesMap))
 	i := 0
@@ -359,13 +444,27 @@ func (cb *ChatBody) ListRoles() []string {
 }
 
 func (cb *ChatBody) MakeStopSlice() []string {
-	namesMap := make(map[string]struct{})
-	for _, m := range cb.Messages {
-		namesMap[m.Role] = struct{}{}
-	}
-	ss := []string{"<|im_end|>"}
-	for k := range namesMap {
-		ss = append(ss, k+":\n")
+	return cb.MakeStopSliceExcluding("", cb.ListRoles())
+}
+
+func (cb *ChatBody) MakeStopSliceExcluding(
+	excludeRole string, roleList []string,
+) []string {
+	ss := []string{}
+	for _, role := range roleList {
+		// Skip the excluded role (typically the current speaker)
+		if role == excludeRole {
+			continue
+		}
+		// Add multiple variations to catch different formatting
+		ss = append(ss,
+			role+":\n",   // Most common: role with newline
+			role+":",     // Role with colon but no newline
+			role+": ",    // Role with colon and single space
+			role+":  ",   // Role with colon and double space (common tokenization)
+			role+":  \n", // Role with colon and double space (common tokenization)
+			role+":   ",  // Role with colon and triple space
+		)
 	}
 	return ss
 }
@@ -420,35 +519,17 @@ type OpenAIReq struct {
 
 // ===
 
-// type LLMModels struct {
-// 	Object string `json:"object"`
-// 	Data   []struct {
-// 		ID      string `json:"id"`
-// 		Object  string `json:"object"`
-// 		Created int    `json:"created"`
-// 		OwnedBy string `json:"owned_by"`
-// 		Meta    struct {
-// 			VocabType int   `json:"vocab_type"`
-// 			NVocab    int   `json:"n_vocab"`
-// 			NCtxTrain int   `json:"n_ctx_train"`
-// 			NEmbd     int   `json:"n_embd"`
-// 			NParams   int64 `json:"n_params"`
-// 			Size      int64 `json:"size"`
-// 		} `json:"meta"`
-// 	} `json:"data"`
-// }
-
 type LlamaCPPReq struct {
 	Model  string `json:"model"`
 	Stream bool   `json:"stream"`
 	// For multimodal requests, prompt should be an object with prompt_string and multimodal_data
 	// For regular requests, prompt is a string
-	Prompt        interface{} `json:"prompt"` // Can be string or object with prompt_string and multimodal_data
-	Temperature   float32     `json:"temperature"`
-	DryMultiplier float32     `json:"dry_multiplier"`
-	Stop          []string    `json:"stop"`
-	MinP          float32     `json:"min_p"`
-	NPredict      int32       `json:"n_predict"`
+	Prompt        any      `json:"prompt"` // Can be string or object with prompt_string and multimodal_data
+	Temperature   float32  `json:"temperature"`
+	DryMultiplier float32  `json:"dry_multiplier"`
+	Stop          []string `json:"stop"`
+	MinP          float32  `json:"min_p"`
+	NPredict      int32    `json:"n_predict"`
 	// MaxTokens        int     `json:"max_tokens"`
 	// DryBase          float64 `json:"dry_base"`
 	// DryAllowedLength int     `json:"dry_allowed_length"`
@@ -476,7 +557,7 @@ type PromptObject struct {
 }
 
 func NewLCPReq(prompt, model string, multimodalData []string, props map[string]float32, stopStrings []string) LlamaCPPReq {
-	var finalPrompt interface{}
+	var finalPrompt any
 	if len(multimodalData) > 0 {
 		// When multimodal data is present, use the object format as per Python example:
 		// { "prompt": { "prompt_string": "...", "multimodal_data": [...] } }
@@ -523,9 +604,22 @@ type LCPModels struct {
 }
 
 func (lcp *LCPModels) ListModels() []string {
-	resp := []string{}
+	resp := make([]string, 0, len(lcp.Data))
 	for _, model := range lcp.Data {
 		resp = append(resp, model.ID)
 	}
 	return resp
 }
+
+type ResponseStats struct {
+	Tokens       int
+	Duration     float64
+	TokensPerSec float64
+}
+
+type ChatRoundReq struct {
+	UserMsg string
+	Role    string
+	Regen   bool
+	Resume  bool
+}

models/openrouter.go

@@ -25,17 +25,23 @@ func NewOpenRouterCompletionReq(model, prompt string, props map[string]float32,
 }
 
 type OpenRouterChatReq struct {
-	Messages    []RoleMsg `json:"messages"`
-	Model       string    `json:"model"`
-	Stream      bool      `json:"stream"`
-	Temperature float32   `json:"temperature"`
-	MinP        float32   `json:"min_p"`
-	NPredict    int32     `json:"max_tokens"`
-	Tools       []Tool    `json:"tools"`
+	Messages    []RoleMsg        `json:"messages"`
+	Model       string           `json:"model"`
+	Stream      bool             `json:"stream"`
+	Temperature float32          `json:"temperature"`
+	MinP        float32          `json:"min_p"`
+	NPredict    int32            `json:"max_tokens"`
+	Tools       []Tool           `json:"tools"`
+	Reasoning   *ReasoningConfig `json:"reasoning,omitempty"`
 }
 
-func NewOpenRouterChatReq(cb ChatBody, props map[string]float32) OpenRouterChatReq {
-	return OpenRouterChatReq{
+type ReasoningConfig struct {
+	Effort  string `json:"effort,omitempty"`  // xhigh, high, medium, low, minimal, none
+	Summary string `json:"summary,omitempty"` // auto, concise, detailed
+}
+
+func NewOpenRouterChatReq(cb ChatBody, props map[string]float32, reasoningEffort string) OpenRouterChatReq {
+	req := OpenRouterChatReq{
 		Messages:    cb.Messages,
 		Model:       cb.Model,
 		Stream:      cb.Stream,
@@ -43,6 +49,13 @@ func NewOpenRouterChatReq(cb ChatBody, props map[string]float32) OpenRouterChatR
 		MinP:        props["min_p"],
 		NPredict:    int32(props["n_predict"]),
 	}
+	// Only include reasoning config if effort is specified and not "none"
+	if reasoningEffort != "" && reasoningEffort != "none" {
+		req.Reasoning = &ReasoningConfig{
+			Effort: reasoningEffort,
+		}
+	}
+	return req
 }
 
 type OpenRouterChatRespNonStream struct {
@@ -82,6 +95,7 @@ type OpenRouterChatResp struct {
 		Delta struct {
 			Role      string          `json:"role"`
 			Content   string          `json:"content"`
+			Reasoning string          `json:"reasoning"`
 			ToolCalls []ToolDeltaResp `json:"tool_calls"`
 		} `json:"delta"`
 		FinishReason       string `json:"finish_reason"`
@@ -143,11 +157,14 @@ type ORModels struct {
 
 func (orm *ORModels) ListModels(free bool) []string {
 	resp := []string{}
-	for _, model := range orm.Data {
+	for i := range orm.Data {
+		model := &orm.Data[i] // Take address of element to avoid copying
 		if free {
-			if model.Pricing.Prompt == "0" && model.Pricing.Request == "0" &&
-				model.Pricing.Completion == "0" {
-				resp = append(resp, model.ID)
+			if model.Pricing.Prompt == "0" && model.Pricing.Completion == "0" {
+				// treat missing request as free
+				if model.Pricing.Request == "" || model.Pricing.Request == "0" {
+					resp = append(resp, model.ID)
+				}
 			}
 		} else {
 			resp = append(resp, model.ID)

models/openrouter_test.go

@@ -0,0 +1,96 @@
+package models
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestORModelsListModels(t *testing.T) {
+	t.Run("unit test with hardcoded data", func(t *testing.T) {
+		jsonData := `{
+			"data": [
+				{
+					"id": "model/free",
+					"pricing": {
+						"prompt": "0",
+						"completion": "0"
+					}
+				},
+				{
+					"id": "model/paid",
+					"pricing": {
+						"prompt": "0.001",
+						"completion": "0.002"
+					}
+				},
+				{
+					"id": "model/request-zero",
+					"pricing": {
+						"prompt": "0",
+						"completion": "0",
+						"request": "0"
+					}
+				},
+				{
+					"id": "model/request-nonzero",
+					"pricing": {
+						"prompt": "0",
+						"completion": "0",
+						"request": "0.5"
+					}
+				}
+			]
+		}`
+		var models ORModels
+		if err := json.Unmarshal([]byte(jsonData), &models); err != nil {
+			t.Fatalf("failed to unmarshal test data: %v", err)
+		}
+		freeModels := models.ListModels(true)
+		if len(freeModels) != 2 {
+			t.Errorf("expected 2 free models, got %d: %v", len(freeModels), freeModels)
+		}
+		expectedFree := map[string]bool{"model/free": true, "model/request-zero": true}
+		for _, id := range freeModels {
+			if !expectedFree[id] {
+				t.Errorf("unexpected free model ID: %s", id)
+			}
+		}
+		allModels := models.ListModels(false)
+		if len(allModels) != 4 {
+			t.Errorf("expected 4 total models, got %d", len(allModels))
+		}
+	})
+	t.Run("integration with or_models.json", func(t *testing.T) {
+		// Attempt to load the real data file from the project root
+		path := filepath.Join("..", "or_models.json")
+		data, err := os.ReadFile(path)
+		if err != nil {
+			t.Skip("or_models.json not found, skipping integration test")
+		}
+		var models ORModels
+		if err := json.Unmarshal(data, &models); err != nil {
+			t.Fatalf("failed to unmarshal %s: %v", path, err)
+		}
+		freeModels := models.ListModels(true)
+		if len(freeModels) == 0 {
+			t.Error("expected at least one free model, got none")
+		}
+		allModels := models.ListModels(false)
+		if len(allModels) == 0 {
+			t.Error("expected at least one model")
+		}
+		// Ensure free models are subset of all models
+		freeSet := make(map[string]bool)
+		for _, id := range freeModels {
+			freeSet[id] = true
+		}
+		for _, id := range freeModels {
+			if !freeSet[id] {
+				t.Errorf("free model %s not found in all models", id)
+			}
+		}
+		t.Logf("found %d free models out of %d total models", len(freeModels), len(allModels))
+	})
+}

pngmeta/altwriter.go

@@ -120,7 +120,7 @@ func createTextChunk(embed PngEmbed) ([]byte, error) {
 	if err := binary.Write(chunk, binary.BigEndian, uint32(len(data))); err != nil {
 		return nil, fmt.Errorf("error writing chunk length: %w", err)
 	}
-	if _, err := chunk.Write([]byte(textChunkType)); err != nil {
+	if _, err := chunk.WriteString(textChunkType); err != nil {
 		return nil, fmt.Errorf("error writing chunk type: %w", err)
 	}
 	if _, err := chunk.Write(data); err != nil {

popups.go

@@ -0,0 +1,519 @@
+package main
+
+import (
+	"gf-lt/models"
+	"slices"
+	"strings"
+
+	"github.com/gdamore/tcell/v2"
+	"github.com/rivo/tview"
+)
+
+// showModelSelectionPopup creates a modal popup to select a model
+func showModelSelectionPopup() {
+	// Helper function to get model list for a given API
+	getModelListForAPI := func(api string) []string {
+		if strings.Contains(api, "api.deepseek.com/") {
+			return []string{"deepseek-chat", "deepseek-reasoner"}
+		} else if strings.Contains(api, "openrouter.ai") {
+			return ORFreeModels
+		}
+		// Assume local llama.cpp - fetch with load status
+		models, err := fetchLCPModelsWithLoadStatus()
+		if err != nil {
+			logger.Error("failed to fetch models with load status", "error", err)
+			return LocalModels
+		}
+		return models
+	}
+	// Get the current model list based on the API
+	modelList := getModelListForAPI(cfg.CurrentAPI)
+	// Check for empty options list
+	if len(modelList) == 0 {
+		logger.Warn("empty model list for", "api", cfg.CurrentAPI, "localModelsLen", len(LocalModels), "orModelsLen", len(ORFreeModels))
+		var message string
+		switch {
+		case strings.Contains(cfg.CurrentAPI, "openrouter.ai"):
+			message = "No OpenRouter models available. Check token and connection."
+		case strings.Contains(cfg.CurrentAPI, "api.deepseek.com"):
+			message = "DeepSeek models should be available. Please report bug."
+		default:
+			message = "No llama.cpp models loaded. Ensure llama.cpp server is running with models."
+		}
+		if err := notifyUser("Empty list", message); err != nil {
+			logger.Error("failed to send notification", "error", err)
+		}
+		return
+	}
+	// Create a list primitive
+	modelListWidget := tview.NewList().ShowSecondaryText(false).
+		SetSelectedBackgroundColor(tcell.ColorGray)
+	modelListWidget.SetTitle("Select Model").SetBorder(true)
+	// Find the current model index to set as selected
+	currentModelIndex := -1
+	for i, model := range modelList {
+		if strings.TrimPrefix(model, models.LoadedMark) == chatBody.Model {
+			currentModelIndex = i
+		}
+		modelListWidget.AddItem(model, "", 0, nil)
+	}
+	// Set the current selection if found
+	if currentModelIndex != -1 {
+		modelListWidget.SetCurrentItem(currentModelIndex)
+	}
+	modelListWidget.SetSelectedFunc(func(index int, mainText string, secondaryText string, shortcut rune) {
+		modelName := strings.TrimPrefix(mainText, models.LoadedMark)
+		chatBody.Model = modelName
+		cfg.CurrentModel = chatBody.Model
+		pages.RemovePage("modelSelectionPopup")
+		app.SetFocus(textArea)
+		updateCachedModelColor()
+		updateStatusLine()
+	})
+	modelListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyEscape {
+			pages.RemovePage("modelSelectionPopup")
+			app.SetFocus(textArea)
+			return nil
+		}
+		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
+			pages.RemovePage("modelSelectionPopup")
+			app.SetFocus(textArea)
+			return nil
+		}
+		return event
+	})
+	modal := func(p tview.Primitive, width, height int) tview.Primitive {
+		return tview.NewFlex().
+			AddItem(nil, 0, 1, false).
+			AddItem(tview.NewFlex().SetDirection(tview.FlexRow).
+				AddItem(nil, 0, 1, false).
+				AddItem(p, height, 1, true).
+				AddItem(nil, 0, 1, false), width, 1, true).
+			AddItem(nil, 0, 1, false)
+	}
+	// Add modal page and make it visible
+	pages.AddPage("modelSelectionPopup", modal(modelListWidget, 80, 20), true, true)
+	app.SetFocus(modelListWidget)
+}
+
+// showAPILinkSelectionPopup creates a modal popup to select an API link
+func showAPILinkSelectionPopup() {
+	// Prepare API links dropdown - ensure current API is in the list, avoid duplicates
+	apiLinks := make([]string, 0, len(cfg.ApiLinks)+1)
+	// Add current API first if it's not already in ApiLinks
+	foundCurrentAPI := false
+	for _, api := range cfg.ApiLinks {
+		if api == cfg.CurrentAPI {
+			foundCurrentAPI = true
+		}
+		apiLinks = append(apiLinks, api)
+	}
+	// If current API is not in the list, add it at the beginning
+	if !foundCurrentAPI {
+		apiLinks = make([]string, 0, len(cfg.ApiLinks)+1)
+		apiLinks = append(apiLinks, cfg.CurrentAPI)
+		apiLinks = append(apiLinks, cfg.ApiLinks...)
+	}
+	// Check for empty options list
+	if len(apiLinks) == 0 {
+		logger.Warn("no API links available for selection")
+		message := "No API links available. Please configure API links in your config file."
+		if err := notifyUser("Empty list", message); err != nil {
+			logger.Error("failed to send notification", "error", err)
+		}
+		return
+	}
+	// Create a list primitive
+	apiListWidget := tview.NewList().ShowSecondaryText(false).
+		SetSelectedBackgroundColor(tcell.ColorGray)
+	apiListWidget.SetTitle("Select API Link").SetBorder(true)
+	// Find the current API index to set as selected
+	currentAPIIndex := -1
+	for i, api := range apiLinks {
+		if api == cfg.CurrentAPI {
+			currentAPIIndex = i
+		}
+		apiListWidget.AddItem(api, "", 0, nil)
+	}
+	// Set the current selection if found
+	if currentAPIIndex != -1 {
+		apiListWidget.SetCurrentItem(currentAPIIndex)
+	}
+	apiListWidget.SetSelectedFunc(func(index int, mainText string, secondaryText string, shortcut rune) {
+		// Update the API in config
+		cfg.CurrentAPI = mainText
+		// Update model list based on new API
+		// Helper function to get model list for a given API (same as in props_table.go)
+		getModelListForAPI := func(api string) []string {
+			if strings.Contains(api, "api.deepseek.com/") {
+				return []string{"deepseek-chat", "deepseek-reasoner"}
+			} else if strings.Contains(api, "openrouter.ai") {
+				return ORFreeModels
+			}
+			// Assume local llama.cpp
+			refreshLocalModelsIfEmpty()
+			localModelsMu.RLock()
+			defer localModelsMu.RUnlock()
+			return LocalModels
+		}
+		newModelList := getModelListForAPI(cfg.CurrentAPI)
+		// Ensure chatBody.Model is in the new list; if not, set to first available model
+		if len(newModelList) > 0 && !slices.Contains(newModelList, chatBody.Model) {
+			chatBody.Model = newModelList[0]
+			cfg.CurrentModel = chatBody.Model
+		}
+		pages.RemovePage("apiLinkSelectionPopup")
+		app.SetFocus(textArea)
+		choseChunkParser()
+		updateCachedModelColor()
+		updateStatusLine()
+	})
+	apiListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyEscape {
+			pages.RemovePage("apiLinkSelectionPopup")
+			app.SetFocus(textArea)
+			return nil
+		}
+		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
+			pages.RemovePage("apiLinkSelectionPopup")
+			app.SetFocus(textArea)
+			return nil
+		}
+		return event
+	})
+	modal := func(p tview.Primitive, width, height int) tview.Primitive {
+		return tview.NewFlex().
+			AddItem(nil, 0, 1, false).
+			AddItem(tview.NewFlex().SetDirection(tview.FlexRow).
+				AddItem(nil, 0, 1, false).
+				AddItem(p, height, 1, true).
+				AddItem(nil, 0, 1, false), width, 1, true).
+			AddItem(nil, 0, 1, false)
+	}
+	// Add modal page and make it visible
+	pages.AddPage("apiLinkSelectionPopup", modal(apiListWidget, 80, 20), true, true)
+	app.SetFocus(apiListWidget)
+}
+
+// showUserRoleSelectionPopup creates a modal popup to select a user role
+func showUserRoleSelectionPopup() {
+	// Get the list of available roles
+	roles := listRolesWithUser()
+	// Check for empty options list
+	if len(roles) == 0 {
+		logger.Warn("no roles available for selection")
+		message := "No roles available for selection."
+		if err := notifyUser("Empty list", message); err != nil {
+			logger.Error("failed to send notification", "error", err)
+		}
+		return
+	}
+	// Create a list primitive
+	roleListWidget := tview.NewList().ShowSecondaryText(false).
+		SetSelectedBackgroundColor(tcell.ColorGray)
+	roleListWidget.SetTitle("Select User Role").SetBorder(true)
+	// Find the current role index to set as selected
+	currentRole := cfg.UserRole
+	if cfg.WriteNextMsgAs != "" {
+		currentRole = cfg.WriteNextMsgAs
+	}
+	currentRoleIndex := -1
+	for i, role := range roles {
+		if strings.EqualFold(role, currentRole) {
+			currentRoleIndex = i
+		}
+		roleListWidget.AddItem(role, "", 0, nil)
+	}
+	// Set the current selection if found
+	if currentRoleIndex != -1 {
+		roleListWidget.SetCurrentItem(currentRoleIndex)
+	}
+	roleListWidget.SetSelectedFunc(func(index int, mainText string, secondaryText string, shortcut rune) {
+		// Update the user role in config
+		cfg.WriteNextMsgAs = mainText
+		// role got switch, update textview with character specific context for user
+		filtered := filterMessagesForCharacter(chatBody.Messages, mainText)
+		textView.SetText(chatToText(filtered, cfg.ShowSys))
+		// Remove the popup page
+		pages.RemovePage("userRoleSelectionPopup")
+		app.SetFocus(textArea)
+		// Update the status line to reflect the change
+		updateStatusLine()
+		colorText()
+	})
+	roleListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyEscape {
+			pages.RemovePage("userRoleSelectionPopup")
+			app.SetFocus(textArea)
+			return nil
+		}
+		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
+			pages.RemovePage("userRoleSelectionPopup")
+			app.SetFocus(textArea)
+			return nil
+		}
+		return event
+	})
+	modal := func(p tview.Primitive, width, height int) tview.Primitive {
+		return tview.NewFlex().
+			AddItem(nil, 0, 1, false).
+			AddItem(tview.NewFlex().SetDirection(tview.FlexRow).
+				AddItem(nil, 0, 1, false).
+				AddItem(p, height, 1, true).
+				AddItem(nil, 0, 1, false), width, 1, true).
+			AddItem(nil, 0, 1, false)
+	}
+	// Add modal page and make it visible
+	pages.AddPage("userRoleSelectionPopup", modal(roleListWidget, 80, 20), true, true)
+	app.SetFocus(roleListWidget)
+}
+
+// showBotRoleSelectionPopup creates a modal popup to select a bot role
+func showBotRoleSelectionPopup() {
+	// Get the list of available roles
+	roles := listChatRoles()
+	if len(roles) == 0 {
+		logger.Warn("empty roles in chat")
+	}
+	if !strInSlice(cfg.AssistantRole, roles) {
+		roles = append(roles, cfg.AssistantRole)
+	}
+	// Check for empty options list
+	if len(roles) == 0 {
+		logger.Warn("no roles available for selection")
+		message := "No roles available for selection."
+		if err := notifyUser("Empty list", message); err != nil {
+			logger.Error("failed to send notification", "error", err)
+		}
+		return
+	}
+	// Create a list primitive
+	roleListWidget := tview.NewList().ShowSecondaryText(false).
+		SetSelectedBackgroundColor(tcell.ColorGray)
+	roleListWidget.SetTitle("Select Bot Role").SetBorder(true)
+	// Find the current role index to set as selected
+	currentRole := cfg.AssistantRole
+	if cfg.WriteNextMsgAsCompletionAgent != "" {
+		currentRole = cfg.WriteNextMsgAsCompletionAgent
+	}
+	currentRoleIndex := -1
+	for i, role := range roles {
+		if strings.EqualFold(role, currentRole) {
+			currentRoleIndex = i
+		}
+		roleListWidget.AddItem(role, "", 0, nil)
+	}
+	// Set the current selection if found
+	if currentRoleIndex != -1 {
+		roleListWidget.SetCurrentItem(currentRoleIndex)
+	}
+	roleListWidget.SetSelectedFunc(func(index int, mainText string, secondaryText string, shortcut rune) {
+		// Update the bot role in config
+		cfg.WriteNextMsgAsCompletionAgent = mainText
+		// Remove the popup page
+		pages.RemovePage("botRoleSelectionPopup")
+		app.SetFocus(textArea)
+		// Update the status line to reflect the change
+		updateStatusLine()
+	})
+	roleListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyEscape {
+			pages.RemovePage("botRoleSelectionPopup")
+			app.SetFocus(textArea)
+			return nil
+		}
+		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
+			pages.RemovePage("botRoleSelectionPopup")
+			app.SetFocus(textArea)
+			return nil
+		}
+		return event
+	})
+	modal := func(p tview.Primitive, width, height int) tview.Primitive {
+		return tview.NewFlex().
+			AddItem(nil, 0, 1, false).
+			AddItem(tview.NewFlex().SetDirection(tview.FlexRow).
+				AddItem(nil, 0, 1, false).
+				AddItem(p, height, 1, true).
+				AddItem(nil, 0, 1, false), width, 1, true).
+			AddItem(nil, 0, 1, false)
+	}
+	// Add modal page and make it visible
+	pages.AddPage("botRoleSelectionPopup", modal(roleListWidget, 80, 20), true, true)
+	app.SetFocus(roleListWidget)
+}
+
+func showShellFileCompletionPopup(filter string) {
+	baseDir := cfg.FilePickerDir
+	if baseDir == "" {
+		baseDir = "."
+	}
+	complMatches := scanFiles(baseDir, filter)
+	if len(complMatches) == 0 {
+		return
+	}
+	if len(complMatches) == 1 {
+		currentText := shellInput.GetText()
+		atIdx := strings.LastIndex(currentText, "@")
+		if atIdx >= 0 {
+			before := currentText[:atIdx]
+			shellInput.SetText(before + complMatches[0])
+		}
+		return
+	}
+	widget := tview.NewList().ShowSecondaryText(false).
+		SetSelectedBackgroundColor(tcell.ColorGray)
+	widget.SetTitle("file completion").SetBorder(true)
+	for _, m := range complMatches {
+		widget.AddItem(m, "", 0, nil)
+	}
+	widget.SetSelectedFunc(func(index int, mainText string, secondaryText string, shortcut rune) {
+		currentText := shellInput.GetText()
+		atIdx := strings.LastIndex(currentText, "@")
+		if atIdx >= 0 {
+			before := currentText[:atIdx]
+			shellInput.SetText(before + mainText)
+		}
+		pages.RemovePage("shellFileCompletionPopup")
+		app.SetFocus(shellInput)
+	})
+	widget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyEscape {
+			pages.RemovePage("shellFileCompletionPopup")
+			app.SetFocus(shellInput)
+			return nil
+		}
+		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
+			pages.RemovePage("shellFileCompletionPopup")
+			app.SetFocus(shellInput)
+			return nil
+		}
+		return event
+	})
+	modal := func(p tview.Primitive, width, height int) tview.Primitive {
+		return tview.NewFlex().
+			AddItem(nil, 0, 1, false).
+			AddItem(tview.NewFlex().SetDirection(tview.FlexRow).
+				AddItem(nil, 0, 1, false).
+				AddItem(p, height, 1, true).
+				AddItem(nil, 0, 1, false), width, 1, true).
+			AddItem(nil, 0, 1, false)
+	}
+	pages.AddPage("shellFileCompletionPopup", modal(widget, 80, 20), true, true)
+	app.SetFocus(widget)
+}
+
+func updateWidgetColors(theme *tview.Theme) {
+	bgColor := theme.PrimitiveBackgroundColor
+	fgColor := theme.PrimaryTextColor
+	borderColor := theme.BorderColor
+	titleColor := theme.TitleColor
+	textView.SetBackgroundColor(bgColor)
+	textView.SetTextColor(fgColor)
+	textView.SetBorderColor(borderColor)
+	textView.SetTitleColor(titleColor)
+	textArea.SetBackgroundColor(bgColor)
+	textArea.SetBorderColor(borderColor)
+	textArea.SetTitleColor(titleColor)
+	textArea.SetTextStyle(tcell.StyleDefault.Background(bgColor).Foreground(fgColor))
+	textArea.SetPlaceholderStyle(tcell.StyleDefault.Background(bgColor).Foreground(fgColor))
+	textArea.SetText(textArea.GetText(), true)
+	editArea.SetBackgroundColor(bgColor)
+	editArea.SetBorderColor(borderColor)
+	editArea.SetTitleColor(titleColor)
+	editArea.SetTextStyle(tcell.StyleDefault.Background(bgColor).Foreground(fgColor))
+	editArea.SetPlaceholderStyle(tcell.StyleDefault.Background(bgColor).Foreground(fgColor))
+	editArea.SetText(editArea.GetText(), true)
+	statusLineWidget.SetBackgroundColor(bgColor)
+	statusLineWidget.SetTextColor(fgColor)
+	statusLineWidget.SetBorderColor(borderColor)
+	statusLineWidget.SetTitleColor(titleColor)
+	helpView.SetBackgroundColor(bgColor)
+	helpView.SetTextColor(fgColor)
+	helpView.SetBorderColor(borderColor)
+	helpView.SetTitleColor(titleColor)
+	searchField.SetBackgroundColor(bgColor)
+	searchField.SetBorderColor(borderColor)
+	searchField.SetTitleColor(titleColor)
+}
+
+// showColorschemeSelectionPopup creates a modal popup to select a colorscheme
+func showColorschemeSelectionPopup() {
+	// Get the list of available colorschemes
+	schemeNames := make([]string, 0, len(colorschemes))
+	for name := range colorschemes {
+		schemeNames = append(schemeNames, name)
+	}
+	slices.Sort(schemeNames)
+	// Check for empty options list
+	if len(schemeNames) == 0 {
+		logger.Warn("no colorschemes available for selection")
+		message := "No colorschemes available."
+		if err := notifyUser("Empty list", message); err != nil {
+			logger.Error("failed to send notification", "error", err)
+		}
+		return
+	}
+	// Create a list primitive
+	schemeListWidget := tview.NewList().ShowSecondaryText(false).
+		SetSelectedBackgroundColor(tcell.ColorGray)
+	schemeListWidget.SetTitle("Select Colorscheme").SetBorder(true)
+	currentScheme := "default"
+	for name := range colorschemes {
+		if tview.Styles == colorschemes[name] {
+			currentScheme = name
+			break
+		}
+	}
+	currentSchemeIndex := -1
+	for i, scheme := range schemeNames {
+		if scheme == currentScheme {
+			currentSchemeIndex = i
+		}
+		schemeListWidget.AddItem(scheme, "", 0, nil)
+	}
+	// Set the current selection if found
+	if currentSchemeIndex != -1 {
+		schemeListWidget.SetCurrentItem(currentSchemeIndex)
+	}
+	schemeListWidget.SetSelectedFunc(func(index int, mainText string, secondaryText string, shortcut rune) {
+		// Update the colorscheme
+		if theme, ok := colorschemes[mainText]; ok {
+			tview.Styles = theme
+			go func() {
+				app.QueueUpdateDraw(func() {
+					updateWidgetColors(&theme)
+				})
+			}()
+		}
+		// Remove the popup page
+		pages.RemovePage("colorschemeSelectionPopup")
+		app.SetFocus(textArea)
+	})
+	schemeListWidget.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyEscape {
+			pages.RemovePage("colorschemeSelectionPopup")
+			app.SetFocus(textArea)
+			return nil
+		}
+		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
+			pages.RemovePage("colorschemeSelectionPopup")
+			app.SetFocus(textArea)
+			return nil
+		}
+		return event
+	})
+	modal := func(p tview.Primitive, width, height int) tview.Primitive {
+		return tview.NewFlex().
+			AddItem(nil, 0, 1, false).
+			AddItem(tview.NewFlex().SetDirection(tview.FlexRow).
+				AddItem(nil, 0, 1, false).
+				AddItem(p, height, 1, true).
+				AddItem(nil, 0, 1, false), width, 1, true).
+			AddItem(nil, 0, 1, false)
+	}
+	// Add modal page and make it visible
+	pages.AddPage("colorschemeSelectionPopup", modal(schemeListWidget, 40, len(schemeNames)+2), true, true)
+	app.SetFocus(schemeListWidget)
+}

props_table.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -53,7 +52,6 @@ func makePropsTable(props map[string]float32) *tview.Table {
 	row++
 	// Store cell data for later use in selection functions
 	cellData := make(map[string]*CellData)
-	var modelCellID string // will be set for the model selection row
 	// Helper function to add a checkbox-like row
 	addCheckboxRow := func(label string, initialValue bool, onChange func(bool)) {
 		table.SetCell(row, 0,
@@ -117,9 +115,6 @@ func makePropsTable(props map[string]float32) *tview.Table {
 		row++
 	}
 	// Add checkboxes
-	addCheckboxRow("Insert <think> tag (/completion only)", cfg.ThinkUse, func(checked bool) {
-		cfg.ThinkUse = checked
-	})
 	addCheckboxRow("RAG use", cfg.RAGEnabled, func(checked bool) {
 		cfg.RAGEnabled = checked
 	})
@@ -137,11 +132,25 @@ func makePropsTable(props map[string]float32) *tview.Table {
 		// Reconfigure the app's mouse setting
 		app.EnableMouse(cfg.EnableMouse)
 	})
+	addCheckboxRow("Image Preview (file picker)", cfg.ImagePreview, func(checked bool) {
+		cfg.ImagePreview = checked
+	})
+	addCheckboxRow("Auto turn (for cards with many chars)", cfg.AutoTurn, func(checked bool) {
+		cfg.AutoTurn = checked
+	})
+	addCheckboxRow("Char specific context", cfg.CharSpecificContextEnabled, func(checked bool) {
+		cfg.CharSpecificContextEnabled = checked
+	})
 	// Add dropdowns
 	logLevels := []string{"Debug", "Info", "Warn"}
 	addListPopupRow("Set log level", logLevels, GetLogLevel(), func(option string) {
 		setLogLevel(option)
 	})
+	// Add reasoning effort dropdown (for OpenRouter and supported APIs)
+	reasoningEfforts := []string{"", "none", "minimal", "low", "medium", "high", "xhigh"}
+	addListPopupRow("Reasoning effort (OR)", reasoningEfforts, cfg.ReasoningEffort, func(option string) {
+		cfg.ReasoningEffort = option
+	})
 	// Helper function to get model list for a given API
 	getModelListForAPI := func(api string) []string {
 		if strings.Contains(api, "api.deepseek.com/") {
@@ -155,52 +164,6 @@ func makePropsTable(props map[string]float32) *tview.Table {
 		defer localModelsMu.RUnlock()
 		return LocalModels
 	}
-	var modelRowIndex int // will be set before model row is added
-	// Prepare API links dropdown - ensure current API is first, avoid duplicates
-	apiLinks := make([]string, 0, len(cfg.ApiLinks)+1)
-	apiLinks = append(apiLinks, cfg.CurrentAPI)
-	for _, api := range cfg.ApiLinks {
-		if api != cfg.CurrentAPI {
-			apiLinks = append(apiLinks, api)
-		}
-	}
-	addListPopupRow("Select an api", apiLinks, cfg.CurrentAPI, func(option string) {
-		cfg.CurrentAPI = option
-		// Update model list based on new API
-		newModelList := getModelListForAPI(cfg.CurrentAPI)
-		if modelCellID != "" {
-			if data := cellData[modelCellID]; data != nil {
-				data.Options = newModelList
-			}
-		}
-		// Ensure chatBody.Model is in the new list; if not, set to first available model
-		if len(newModelList) > 0 && !slices.Contains(newModelList, chatBody.Model) {
-			chatBody.Model = newModelList[0]
-			cfg.CurrentModel = chatBody.Model
-			// Update the displayed cell text - need to find model row
-			// Search for model row by label
-			for r := 0; r < table.GetRowCount(); r++ {
-				if cell := table.GetCell(r, 0); cell != nil && cell.Text == "Select a model" {
-					if valueCell := table.GetCell(r, 1); valueCell != nil {
-						valueCell.SetText(chatBody.Model)
-					}
-					break
-				}
-			}
-		}
-	})
-	// Prepare model list dropdown
-	modelRowIndex = row
-	modelCellID = fmt.Sprintf("listpopup_%d", modelRowIndex)
-	modelList := getModelListForAPI(cfg.CurrentAPI)
-	addListPopupRow("Select a model", modelList, chatBody.Model, func(option string) {
-		chatBody.Model = option
-		cfg.CurrentModel = chatBody.Model
-	})
-	// Role selection dropdown
-	addListPopupRow("Write next message as", listRolesWithUser(), cfg.WriteNextMsgAs, func(option string) {
-		cfg.WriteNextMsgAs = option
-	})
 	// Add input fields
 	addInputRow("New char to write msg as", "", func(text string) {
 		if text != "" {
@@ -307,11 +270,12 @@ func makePropsTable(props map[string]float32) *tview.Table {
 					logger.Warn("empty options list for", "label", label, "api", cfg.CurrentAPI, "localModelsLen", len(LocalModels), "orModelsLen", len(ORFreeModels))
 					message := "No options available for " + label
 					if label == "Select a model" {
-						if strings.Contains(cfg.CurrentAPI, "openrouter.ai") {
+						switch {
+						case strings.Contains(cfg.CurrentAPI, "openrouter.ai"):
 							message = "No OpenRouter models available. Check token and connection."
-						} else if strings.Contains(cfg.CurrentAPI, "api.deepseek.com") {
+						case strings.Contains(cfg.CurrentAPI, "api.deepseek.com"):
 							message = "DeepSeek models should be available. Please report bug."
-						} else {
+						default:
 							message = "No llama.cpp models loaded. Ensure llama.cpp server is running with models."
 						}
 					}

rag/embedder.go

@@ -131,7 +131,6 @@ func (a *APIEmbedder) EmbedSlice(lines []string) ([][]float32, error) {
 		}
 		embeddings[data.Index] = data.Embedding
 	}
-
 	return embeddings, nil
 }
 

rag/extractors.go

@@ -0,0 +1,181 @@
+package rag
+
+import (
+	"archive/zip"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/ledongthuc/pdf"
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/parser"
+	"github.com/yuin/goldmark/renderer/html"
+)
+
+func ExtractText(fpath string) (string, error) {
+	ext := strings.ToLower(path.Ext(fpath))
+	switch ext {
+	case ".txt":
+		return extractTextFromFile(fpath)
+	case ".md", ".markdown":
+		return extractTextFromMarkdown(fpath)
+	case ".html", ".htm":
+		return extractTextFromHtmlFile(fpath)
+	case ".epub":
+		return extractTextFromEpub(fpath)
+	case ".pdf":
+		return extractTextFromPdf(fpath)
+	default:
+		return "", fmt.Errorf("unsupported file format: %s", ext)
+	}
+}
+
+func extractTextFromFile(fpath string) (string, error) {
+	data, err := os.ReadFile(fpath)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
+func extractTextFromHtmlFile(fpath string) (string, error) {
+	data, err := os.ReadFile(fpath)
+	if err != nil {
+		return "", err
+	}
+	return extractTextFromHtmlContent(data)
+}
+
+// non utf-8 encoding?
+func extractTextFromHtmlContent(data []byte) (string, error) {
+	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(data))
+	if err != nil {
+		return "", err
+	}
+	// Remove script and style tags
+	doc.Find("script, style, noscript").Each(func(i int, s *goquery.Selection) {
+		s.Remove()
+	})
+	// Get text and clean it
+	text := doc.Text()
+	// Collapse all whitespace (newlines, tabs, multiple spaces) into single spaces
+	cleaned := strings.Join(strings.Fields(text), " ")
+	return cleaned, nil
+}
+
+func extractTextFromMarkdown(fpath string) (string, error) {
+	data, err := os.ReadFile(fpath)
+	if err != nil {
+		return "", err
+	}
+	// Convert markdown to HTML
+	md := goldmark.New(
+		goldmark.WithExtensions(extension.GFM),
+		goldmark.WithParserOptions(parser.WithAutoHeadingID()),
+		goldmark.WithRendererOptions(html.WithUnsafe()), // allow raw HTML if needed
+	)
+	var buf bytes.Buffer
+	if err := md.Convert(data, &buf); err != nil {
+		return "", err
+	}
+	// Now extract text from the resulting HTML (using goquery or similar)
+	return extractTextFromHtmlContent(buf.Bytes())
+}
+
+func extractTextFromEpub(fpath string) (string, error) {
+	r, err := zip.OpenReader(fpath)
+	if err != nil {
+		return "", fmt.Errorf("failed to open epub: %w", err)
+	}
+	defer r.Close()
+	var sb strings.Builder
+	for _, f := range r.File {
+		ext := strings.ToLower(path.Ext(f.Name))
+		if ext != ".xhtml" && ext != ".html" && ext != ".htm" && ext != ".xml" {
+			continue
+		}
+
+		// Skip manifest, toc, ncx files - they don't contain book content
+		nameLower := strings.ToLower(f.Name)
+		if strings.Contains(nameLower, "toc") || strings.Contains(nameLower, "nav") ||
+			strings.Contains(nameLower, "manifest") || strings.Contains(nameLower, ".opf") ||
+			strings.HasSuffix(nameLower, ".ncx") {
+			continue
+		}
+
+		rc, err := f.Open()
+		if err != nil {
+			continue
+		}
+
+		if sb.Len() > 0 {
+			sb.WriteString("\n\n")
+		}
+		sb.WriteString(f.Name)
+		sb.WriteString("\n")
+
+		buf, readErr := io.ReadAll(rc)
+		rc.Close()
+		if readErr == nil {
+			sb.WriteString(stripHTML(string(buf)))
+		}
+	}
+	if sb.Len() == 0 {
+		return "", errors.New("no content extracted from epub")
+	}
+	return sb.String(), nil
+}
+
+func stripHTML(html string) string {
+	var sb strings.Builder
+	inTag := false
+	for _, r := range html {
+		switch r {
+		case '<':
+			inTag = true
+		case '>':
+			inTag = false
+		default:
+			if !inTag {
+				sb.WriteRune(r)
+			}
+		}
+	}
+	return sb.String()
+}
+
+func extractTextFromPdf(fpath string) (string, error) {
+	_, err := exec.LookPath("pdftotext")
+	if err == nil {
+		out, err := exec.Command("pdftotext", "-layout", fpath, "-").Output()
+		if err == nil && len(out) > 0 {
+			return string(out), nil
+		}
+	}
+	return extractTextFromPdfPureGo(fpath)
+}
+
+func extractTextFromPdfPureGo(fpath string) (string, error) {
+	df, r, err := pdf.Open(fpath)
+	if err != nil {
+		return "", fmt.Errorf("failed to open pdf: %w", err)
+	}
+	defer df.Close()
+	textReader, err := r.GetPlainText()
+	if err != nil {
+		return "", fmt.Errorf("failed to extract text from pdf: %w", err)
+	}
+	var buf bytes.Buffer
+	_, err = io.Copy(&buf, textReader)
+	if err != nil {
+		return "", fmt.Errorf("failed to read pdf text: %w", err)
+	}
+	return buf.String(), nil
+}

rag/rag.go

@@ -7,8 +7,9 @@ import (
 	"gf-lt/models"
 	"gf-lt/storage"
 	"log/slog"
-	"os"
 	"path"
+	"regexp"
+	"sort"
 	"strings"
 	"sync"
 
@@ -23,19 +24,18 @@ var (
 	ErrRAGStatus        = "some error occurred; failed to transfer data to vector db"
 )
 
-
 type RAG struct {
 	logger   *slog.Logger
 	store    storage.FullRepo
 	cfg      *config.Config
 	embedder Embedder
 	storage  *VectorStorage
+	mu       sync.Mutex
 }
 
 func New(l *slog.Logger, s storage.FullRepo, cfg *config.Config) *RAG {
 	// Initialize with API embedder by default, could be configurable later
 	embedder := NewAPIEmbedder(l, cfg)
-
 	rag := &RAG{
 		logger:   l,
 		store:    s,
@@ -54,7 +54,9 @@ func wordCounter(sentence string) int {
 }
 
 func (r *RAG) LoadRAG(fpath string) error {
-	data, err := os.ReadFile(fpath)
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	fileText, err := ExtractText(fpath)
 	if err != nil {
 		return err
 	}
@@ -63,10 +65,7 @@ func (r *RAG) LoadRAG(fpath string) error {
 	case LongJobStatusCh <- LoadedFileRAGStatus:
 	default:
 		r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", LoadedFileRAGStatus)
-		// Channel is full or closed, ignore the message to prevent panic
 	}
-
-	fileText := string(data)
 	tokenizer, err := english.NewSentenceTokenizer(nil)
 	if err != nil {
 		return err
@@ -76,19 +75,16 @@ func (r *RAG) LoadRAG(fpath string) error {
 	for i, s := range sentences {
 		sents[i] = s.Text
 	}
-
 	// Group sentences into paragraphs based on word limit
 	paragraphs := []string{}
 	par := strings.Builder{}
 	for i := 0; i < len(sents); i++ {
-		// Only add sentences that aren't empty
 		if strings.TrimSpace(sents[i]) != "" {
 			if par.Len() > 0 {
-				par.WriteString(" ") // Add space between sentences
+				par.WriteString(" ")
 			}
 			par.WriteString(sents[i])
 		}
-
 		if wordCounter(par.String()) > int(r.cfg.RAGWordLimit) {
 			paragraph := strings.TrimSpace(par.String())
 			if paragraph != "" {
@@ -97,7 +93,6 @@ func (r *RAG) LoadRAG(fpath string) error {
 			par.Reset()
 		}
 	}
-
 	// Handle any remaining content in the paragraph buffer
 	if par.Len() > 0 {
 		paragraph := strings.TrimSpace(par.String())
@@ -105,215 +100,82 @@ func (r *RAG) LoadRAG(fpath string) error {
 			paragraphs = append(paragraphs, paragraph)
 		}
 	}
-
 	// Adjust batch size if needed
-	if len(paragraphs) < int(r.cfg.RAGBatchSize) && len(paragraphs) > 0 {
+	if len(paragraphs) < r.cfg.RAGBatchSize && len(paragraphs) > 0 {
 		r.cfg.RAGBatchSize = len(paragraphs)
 	}
-
 	if len(paragraphs) == 0 {
 		return errors.New("no valid paragraphs found in file")
 	}
-
-	var (
-		maxChSize = 100
-		left      = 0
-		right     = r.cfg.RAGBatchSize
-		batchCh   = make(chan map[int][]string, maxChSize)
-		vectorCh  = make(chan []models.VectorRow, maxChSize)
-		errCh     = make(chan error, 1)
-		wg        = new(sync.WaitGroup)
-		lock      = new(sync.Mutex)
-	)
-
-	defer close(errCh)
-	defer close(batchCh)
-
-	// Fill input channel with batches
-	ctn := 0
-	totalParagraphs := len(paragraphs)
-	for {
-		if int(right) > totalParagraphs {
-			batchCh <- map[int][]string{left: paragraphs[left:]}
-			break
+	// Process paragraphs in batches synchronously
+	batchCount := 0
+	for i := 0; i < len(paragraphs); i += r.cfg.RAGBatchSize {
+		end := i + r.cfg.RAGBatchSize
+		if end > len(paragraphs) {
+			end = len(paragraphs)
 		}
-		batchCh <- map[int][]string{left: paragraphs[left:right]}
-		left, right = right, right+r.cfg.RAGBatchSize
-		ctn++
-	}
-
-	finishedBatchesMsg := fmt.Sprintf("finished batching batches#: %d; paragraphs: %d; sentences: %d\n", ctn+1, len(paragraphs), len(sents))
-	r.logger.Debug(finishedBatchesMsg)
-	select {
-	case LongJobStatusCh <- finishedBatchesMsg:
-	default:
-		r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", finishedBatchesMsg)
-		// Channel is full or closed, ignore the message to prevent panic
-	}
-
-	// Start worker goroutines with WaitGroup
-	wg.Add(int(r.cfg.RAGWorkers))
-	for w := 0; w < int(r.cfg.RAGWorkers); w++ {
-		go func(workerID int) {
-			defer wg.Done()
-			r.batchToVectorAsync(lock, workerID, batchCh, vectorCh, errCh, path.Base(fpath))
-		}(w)
-	}
-
-	// Use a goroutine to close the batchCh when all batches are sent
-	go func() {
-		wg.Wait()
-		close(vectorCh) // Close vectorCh when all workers are done
-	}()
-
-	// Check for errors from workers
-	// Use a non-blocking check for errors
-	select {
-	case err := <-errCh:
+		batch := paragraphs[i:end]
+		batchCount++
+		// Filter empty paragraphs
+		nonEmptyBatch := make([]string, 0, len(batch))
+		for _, p := range batch {
+			if strings.TrimSpace(p) != "" {
+				nonEmptyBatch = append(nonEmptyBatch, strings.TrimSpace(p))
+			}
+		}
+		if len(nonEmptyBatch) == 0 {
+			continue
+		}
+		// Embed the batch
+		embeddings, err := r.embedder.EmbedSlice(nonEmptyBatch)
 		if err != nil {
-			r.logger.Error("error during RAG processing", "error", err)
+			r.logger.Error("failed to embed batch", "error", err, "batch", batchCount)
+			select {
+			case LongJobStatusCh <- ErrRAGStatus:
+			default:
+				r.logger.Warn("LongJobStatusCh channel full, dropping message")
+			}
+			return fmt.Errorf("failed to embed batch %d: %w", batchCount, err)
+		}
+		if len(embeddings) != len(nonEmptyBatch) {
+			err := errors.New("embedding count mismatch")
+			r.logger.Error("embedding mismatch", "expected", len(nonEmptyBatch), "got", len(embeddings))
 			return err
 		}
-	default:
-		// No immediate error, continue
-	}
-
-	// Write vectors to storage - this will block until vectorCh is closed
-	return r.writeVectors(vectorCh)
-}
-
-func (r *RAG) writeVectors(vectorCh chan []models.VectorRow) error {
-	for {
-		for batch := range vectorCh {
-			for _, vector := range batch {
-				if err := r.storage.WriteVector(&vector); err != nil {
-					r.logger.Error("failed to write vector to DB", "error", err, "slug", vector.Slug)
-					select {
-					case LongJobStatusCh <- ErrRAGStatus:
-					default:
-						r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", ErrRAGStatus)
-						// Channel is full or closed, ignore the message to prevent panic
-					}
-					return err // Stop the entire RAG operation on DB error
-				}
+		// Write vectors to storage
+		filename := path.Base(fpath)
+		for j, text := range nonEmptyBatch {
+			vector := models.VectorRow{
+				Embeddings: embeddings[j],
+				RawText:    text,
+				Slug:       fmt.Sprintf("%s_%d_%d", filename, batchCount, j),
+				FileName:   filename,
 			}
-			r.logger.Debug("wrote batch to db", "size", len(batch), "vector_chan_len", len(vectorCh))
-			if len(vectorCh) == 0 {
-				r.logger.Debug("finished writing vectors")
+			if err := r.storage.WriteVector(&vector); err != nil {
+				r.logger.Error("failed to write vector to DB", "error", err, "slug", vector.Slug)
 				select {
-				case LongJobStatusCh <- FinishedRAGStatus:
+				case LongJobStatusCh <- ErrRAGStatus:
 				default:
-					r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", FinishedRAGStatus)
-					// Channel is full or closed, ignore the message to prevent panic
+					r.logger.Warn("LongJobStatusCh channel full, dropping message")
 				}
-				return nil
+				return fmt.Errorf("failed to write vector: %w", err)
 			}
 		}
-	}
-}
-
-func (r *RAG) batchToVectorAsync(lock *sync.Mutex, id int, inputCh <-chan map[int][]string,
-	vectorCh chan<- []models.VectorRow, errCh chan error, filename string) {
-	var err error
-
-	defer func() {
-		// For errCh, make sure we only send if there's actually an error and the channel can accept it
-		if err != nil {
-			select {
-			case errCh <- err:
-			default:
-				// errCh might be full or closed, log but don't panic
-				r.logger.Warn("errCh channel full or closed, skipping error propagation", "worker", id, "error", err)
-			}
-		}
-	}()
-
-	for {
-		lock.Lock()
-		if len(inputCh) == 0 {
-			lock.Unlock()
-			return
-		}
-
-		select {
-		case linesMap := <-inputCh:
-			for leftI, lines := range linesMap {
-				if err := r.fetchEmb(lines, errCh, vectorCh, fmt.Sprintf("%s_%d", filename, leftI), filename); err != nil {
-					r.logger.Error("error fetching embeddings", "error", err, "worker", id)
-					lock.Unlock()
-					return
-				}
-			}
-			lock.Unlock()
-		case err = <-errCh:
-			r.logger.Error("got an error from error channel", "error", err)
-			lock.Unlock()
-			return
-		default:
-			lock.Unlock()
-		}
-
-		r.logger.Debug("processed batch", "batches#", len(inputCh), "worker#", id)
-		statusMsg := fmt.Sprintf("converted to vector; batches: %d, worker#: %d", len(inputCh), id)
+		r.logger.Debug("wrote batch to db", "batch", batchCount, "size", len(nonEmptyBatch))
+		// Send progress status
+		statusMsg := fmt.Sprintf("processed batch %d/%d", batchCount, (len(paragraphs)+r.cfg.RAGBatchSize-1)/r.cfg.RAGBatchSize)
 		select {
 		case LongJobStatusCh <- statusMsg:
 		default:
-			r.logger.Warn("LongJobStatusCh channel full or closed, dropping status message", "message", statusMsg)
-			// Channel is full or closed, ignore the message to prevent panic
+			r.logger.Warn("LongJobStatusCh channel full, dropping message")
 		}
 	}
-}
-
-func (r *RAG) fetchEmb(lines []string, errCh chan error, vectorCh chan<- []models.VectorRow, slug, filename string) error {
-	// Filter out empty lines before sending to embedder
-	nonEmptyLines := make([]string, 0, len(lines))
-	for _, line := range lines {
-		trimmed := strings.TrimSpace(line)
-		if trimmed != "" {
-			nonEmptyLines = append(nonEmptyLines, trimmed)
-		}
+	r.logger.Debug("finished writing vectors", "batches", batchCount)
+	select {
+	case LongJobStatusCh <- FinishedRAGStatus:
+	default:
+		r.logger.Warn("LongJobStatusCh channel is full or closed, dropping status message", "message", FinishedRAGStatus)
 	}
-
-	// Skip if no non-empty lines
-	if len(nonEmptyLines) == 0 {
-		// Send empty result but don't error
-		vectorCh <- []models.VectorRow{}
-		return nil
-	}
-
-	embeddings, err := r.embedder.EmbedSlice(nonEmptyLines)
-	if err != nil {
-		r.logger.Error("failed to embed lines", "err", err.Error())
-		errCh <- err
-		return err
-	}
-
-	if len(embeddings) == 0 {
-		err := errors.New("no embeddings returned")
-		r.logger.Error("empty embeddings")
-		errCh <- err
-		return err
-	}
-
-	if len(embeddings) != len(nonEmptyLines) {
-		err := errors.New("mismatch between number of lines and embeddings returned")
-		r.logger.Error("embedding mismatch", "err", err.Error())
-		errCh <- err
-		return err
-	}
-
-	// Create a VectorRow for each line in the batch
-	vectors := make([]models.VectorRow, len(nonEmptyLines))
-	for i, line := range nonEmptyLines {
-		vectors[i] = models.VectorRow{
-			Embeddings: embeddings[i],
-			RawText:    line,
-			Slug:       fmt.Sprintf("%s_%d", slug, i),
-			FileName:   filename,
-		}
-	}
-
-	vectorCh <- vectors
 	return nil
 }
 
@@ -332,3 +194,259 @@ func (r *RAG) ListLoaded() ([]string, error) {
 func (r *RAG) RemoveFile(filename string) error {
 	return r.storage.RemoveEmbByFileName(filename)
 }
+
+var (
+	queryRefinementPattern = regexp.MustCompile(`(?i)(based on my (vector db|vector db|vector database|rags?|past (conversations?|chat|messages?))|from my (files?|documents?|data|information|memory)|search (in|my) (vector db|database|rags?)|rag search for)`)
+	importantKeywords      = []string{"project", "architecture", "code", "file", "chat", "conversation", "topic", "summary", "details", "history", "previous", "my", "user", "me"}
+	stopWords              = []string{"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", "from", "up", "down", "left", "right"}
+)
+
+func (r *RAG) RefineQuery(query string) string {
+	original := query
+	query = strings.TrimSpace(query)
+	if len(query) == 0 {
+		return original
+	}
+	if len(query) <= 3 {
+		return original
+	}
+	query = strings.ToLower(query)
+	for _, stopWord := range stopWords {
+		wordPattern := `\b` + stopWord + `\b`
+		re := regexp.MustCompile(wordPattern)
+		query = re.ReplaceAllString(query, "")
+	}
+	query = strings.TrimSpace(query)
+	if len(query) < 5 {
+		return original
+	}
+	if queryRefinementPattern.MatchString(original) {
+		cleaned := queryRefinementPattern.ReplaceAllString(original, "")
+		cleaned = strings.TrimSpace(cleaned)
+		if len(cleaned) >= 5 {
+			return cleaned
+		}
+	}
+	query = r.extractImportantPhrases(query)
+	if len(query) < 5 {
+		return original
+	}
+	return query
+}
+
+func (r *RAG) extractImportantPhrases(query string) string {
+	words := strings.Fields(query)
+	var important []string
+	for _, word := range words {
+		word = strings.Trim(word, ".,!?;:'\"()[]{}")
+		isImportant := false
+		for _, kw := range importantKeywords {
+			if strings.Contains(strings.ToLower(word), kw) {
+				isImportant = true
+				break
+			}
+		}
+		if isImportant || len(word) > 3 {
+			important = append(important, word)
+		}
+	}
+	if len(important) == 0 {
+		return query
+	}
+	return strings.Join(important, " ")
+}
+
+func (r *RAG) GenerateQueryVariations(query string) []string {
+	variations := []string{query}
+	if len(query) < 5 {
+		return variations
+	}
+	parts := strings.Fields(query)
+	if len(parts) == 0 {
+		return variations
+	}
+	if len(parts) >= 2 {
+		trimmed := strings.Join(parts[:len(parts)-1], " ")
+		if len(trimmed) >= 5 {
+			variations = append(variations, trimmed)
+		}
+	}
+	if len(parts) >= 2 {
+		trimmed := strings.Join(parts[1:], " ")
+		if len(trimmed) >= 5 {
+			variations = append(variations, trimmed)
+		}
+	}
+	if !strings.HasSuffix(query, " explanation") {
+		variations = append(variations, query+" explanation")
+	}
+	if !strings.HasPrefix(query, "what is ") {
+		variations = append(variations, "what is "+query)
+	}
+	if !strings.HasSuffix(query, " details") {
+		variations = append(variations, query+" details")
+	}
+	if !strings.HasSuffix(query, " summary") {
+		variations = append(variations, query+" summary")
+	}
+	return variations
+}
+
+func (r *RAG) RerankResults(results []models.VectorRow, query string) []models.VectorRow {
+	type scoredResult struct {
+		row      models.VectorRow
+		distance float32
+	}
+	scored := make([]scoredResult, 0, len(results))
+	for i := range results {
+		row := results[i]
+
+		score := float32(0)
+		rawTextLower := strings.ToLower(row.RawText)
+		queryLower := strings.ToLower(query)
+		if strings.Contains(rawTextLower, queryLower) {
+			score += 10
+		}
+		queryWords := strings.Fields(queryLower)
+		matchCount := 0
+		for _, word := range queryWords {
+			if len(word) > 2 && strings.Contains(rawTextLower, word) {
+				matchCount++
+			}
+		}
+		if len(queryWords) > 0 {
+			score += float32(matchCount) / float32(len(queryWords)) * 5
+		}
+		if row.FileName == "chat" || strings.Contains(strings.ToLower(row.FileName), "conversation") {
+			score += 3
+		}
+		distance := row.Distance - score/100
+		scored = append(scored, scoredResult{row: row, distance: distance})
+	}
+	sort.Slice(scored, func(i, j int) bool {
+		return scored[i].distance < scored[j].distance
+	})
+	unique := make([]models.VectorRow, 0)
+	seen := make(map[string]bool)
+	for i := range scored {
+		if !seen[scored[i].row.Slug] {
+			seen[scored[i].row.Slug] = true
+			unique = append(unique, scored[i].row)
+		}
+	}
+	if len(unique) > 10 {
+		unique = unique[:10]
+	}
+	return unique
+}
+
+func (r *RAG) SynthesizeAnswer(results []models.VectorRow, query string) (string, error) {
+	if len(results) == 0 {
+		return "No relevant information found in the vector database.", nil
+	}
+	var contextBuilder strings.Builder
+	contextBuilder.WriteString("User Query: ")
+	contextBuilder.WriteString(query)
+	contextBuilder.WriteString("\n\nRetrieved Context:\n")
+	for i, row := range results {
+		fmt.Fprintf(&contextBuilder, "[Source %d: %s]\n", i+1, row.FileName)
+		contextBuilder.WriteString(row.RawText)
+		contextBuilder.WriteString("\n\n")
+	}
+	contextBuilder.WriteString("Instructions: ")
+	contextBuilder.WriteString("Based on the retrieved context above, provide a concise, coherent answer to the user's query. ")
+	contextBuilder.WriteString("Extract only the most relevant information. ")
+	contextBuilder.WriteString("If no relevant information is found, state that clearly. ")
+	contextBuilder.WriteString("Cite sources by filename when relevant. ")
+	contextBuilder.WriteString("Do not include unnecessary preamble or explanations.")
+	synthesisPrompt := contextBuilder.String()
+	emb, err := r.LineToVector(synthesisPrompt)
+	if err != nil {
+		r.logger.Error("failed to embed synthesis prompt", "error", err)
+		return "", err
+	}
+	embResp := &models.EmbeddingResp{
+		Embedding: emb,
+		Index:     0,
+	}
+	topResults, err := r.SearchEmb(embResp)
+	if err != nil {
+		r.logger.Error("failed to search for synthesis context", "error", err)
+		return "", err
+	}
+	if len(topResults) > 0 && topResults[0].RawText != synthesisPrompt {
+		return topResults[0].RawText, nil
+	}
+	var finalAnswer strings.Builder
+	finalAnswer.WriteString("Based on the retrieved context:\n\n")
+	for i, row := range results {
+		if i >= 5 {
+			break
+		}
+		fmt.Fprintf(&finalAnswer, "- From %s: %s\n", row.FileName, truncateString(row.RawText, 200))
+	}
+	return finalAnswer.String(), nil
+}
+
+func truncateString(s string, maxLen int) string {
+	if len(s) <= maxLen {
+		return s
+	}
+	return s[:maxLen] + "..."
+}
+
+func (r *RAG) Search(query string, limit int) ([]models.VectorRow, error) {
+	refined := r.RefineQuery(query)
+	variations := r.GenerateQueryVariations(refined)
+	allResults := make([]models.VectorRow, 0)
+	seen := make(map[string]bool)
+	for _, q := range variations {
+		emb, err := r.LineToVector(q)
+		if err != nil {
+			r.logger.Error("failed to embed query variation", "error", err, "query", q)
+			continue
+		}
+
+		embResp := &models.EmbeddingResp{
+			Embedding: emb,
+			Index:     0,
+		}
+
+		results, err := r.SearchEmb(embResp)
+		if err != nil {
+			r.logger.Error("failed to search embeddings", "error", err, "query", q)
+			continue
+		}
+
+		for _, row := range results {
+			if !seen[row.Slug] {
+				seen[row.Slug] = true
+				allResults = append(allResults, row)
+			}
+		}
+	}
+	reranked := r.RerankResults(allResults, query)
+	if len(reranked) > limit {
+		reranked = reranked[:limit]
+	}
+	return reranked, nil
+}
+
+var (
+	ragInstance *RAG
+	ragOnce     sync.Once
+)
+
+func Init(c *config.Config, l *slog.Logger, s storage.FullRepo) error {
+	ragOnce.Do(func() {
+		if c == nil || l == nil || s == nil {
+			return
+		}
+		ragInstance = New(l, s, c)
+	})
+	return nil
+}
+
+func GetInstance() *RAG {
+	return ragInstance
+}

rag/storage.go

@@ -28,7 +28,6 @@ func NewVectorStorage(logger *slog.Logger, store storage.FullRepo) *VectorStorag
 	}
 }
 
-
 // SerializeVector converts []float32 to binary blob
 func SerializeVector(vec []float32) []byte {
 	buf := make([]byte, len(vec)*4) // 4 bytes per float32
@@ -66,17 +65,14 @@ func (vs *VectorStorage) WriteVector(row *models.VectorRow) error {
 
 	// Serialize the embeddings to binary
 	serializedEmbeddings := SerializeVector(row.Embeddings)
-
 	query := fmt.Sprintf(
 		"INSERT INTO %s (embeddings, slug, raw_text, filename) VALUES (?, ?, ?, ?)",
 		tableName,
 	)
-
 	if _, err := vs.sqlxDB.Exec(query, serializedEmbeddings, row.Slug, row.RawText, row.FileName); err != nil {
 		vs.logger.Error("failed to write vector", "error", err, "slug", row.Slug)
 		return err
 	}
-
 	return nil
 }
 
@@ -86,20 +82,18 @@ func (vs *VectorStorage) getTableName(emb []float32) (string, error) {
 
 	// Check if we support this embedding size
 	supportedSizes := map[int]bool{
-		384:   true,
-		768:   true,
-		1024:  true,
-		1536:  true,
-		2048:  true,
-		3072:  true,
-		4096:  true,
-		5120:  true,
+		384:  true,
+		768:  true,
+		1024: true,
+		1536: true,
+		2048: true,
+		3072: true,
+		4096: true,
+		5120: true,
 	}
-
 	if supportedSizes[size] {
 		return fmt.Sprintf("embeddings_%d", size), nil
 	}
-
 	return "", fmt.Errorf("no table for embedding size of %d", size)
 }
 
@@ -126,9 +120,7 @@ func (vs *VectorStorage) SearchClosest(query []float32) ([]models.VectorRow, err
 		vector   models.VectorRow
 		distance float32
 	}
-
 	var topResults []SearchResult
-
 	// Process vectors one by one to avoid loading everything into memory
 	for rows.Next() {
 		var (
@@ -176,14 +168,12 @@ func (vs *VectorStorage) SearchClosest(query []float32) ([]models.VectorRow, err
 		result.vector.Distance = result.distance
 		results = append(results, result.vector)
 	}
-
 	return results, nil
 }
 
 // ListFiles returns a list of all loaded files
 func (vs *VectorStorage) ListFiles() ([]string, error) {
 	fileLists := make([][]string, 0)
-
 	// Query all supported tables and combine results
 	embeddingSizes := []int{384, 768, 1024, 1536, 2048, 3072, 4096, 5120}
 	for _, size := range embeddingSizes {
@@ -219,14 +209,12 @@ func (vs *VectorStorage) ListFiles() ([]string, error) {
 			}
 		}
 	}
-
 	return allFiles, nil
 }
 
 // RemoveEmbByFileName removes all embeddings associated with a specific filename
 func (vs *VectorStorage) RemoveEmbByFileName(filename string) error {
 	var errors []string
-
 	embeddingSizes := []int{384, 768, 1024, 1536, 2048, 3072, 4096, 5120}
 	for _, size := range embeddingSizes {
 		table := fmt.Sprintf("embeddings_%d", size)
@@ -235,11 +223,9 @@ func (vs *VectorStorage) RemoveEmbByFileName(filename string) error {
 			errors = append(errors, err.Error())
 		}
 	}
-
 	if len(errors) > 0 {
 		return fmt.Errorf("errors occurred: %s", strings.Join(errors, "; "))
 	}
-
 	return nil
 }
 
@@ -248,18 +234,15 @@ func cosineSimilarity(a, b []float32) float32 {
 	if len(a) != len(b) {
 		return 0.0
 	}
-
 	var dotProduct, normA, normB float32
 	for i := 0; i < len(a); i++ {
 		dotProduct += a[i] * b[i]
 		normA += a[i] * a[i]
 		normB += b[i] * b[i]
 	}
-
 	if normA == 0 || normB == 0 {
 		return 0.0
 	}
-
 	return dotProduct / (sqrt(normA) * sqrt(normB))
 }
 
@@ -275,4 +258,3 @@ func sqrt(f float32) float32 {
 	}
 	return guess
 }
-

server.go

@@ -1,74 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"gf-lt/config"
-	"net/http"
-	"time"
-)
-
-type Server struct {
-	// nolint
-	config config.Config
-}
-
-func (srv *Server) ListenToRequests(port string) {
-	// h := srv.actions
-	mux := http.NewServeMux()
-	server := &http.Server{
-		Addr:         "localhost:" + port,
-		Handler:      mux,
-		ReadTimeout:  time.Second * 5,
-		WriteTimeout: time.Second * 5,
-	}
-	mux.HandleFunc("GET /ping", pingHandler)
-	mux.HandleFunc("GET /model", modelHandler)
-	mux.HandleFunc("POST /completion", completionHandler)
-	fmt.Println("Listening", "addr", server.Addr)
-	if err := server.ListenAndServe(); err != nil {
-		panic(err)
-	}
-}
-
-// create server
-// listen to the completion endpoint handler
-func pingHandler(w http.ResponseWriter, req *http.Request) {
-	if _, err := w.Write([]byte("pong")); err != nil {
-		logger.Error("server ping", "error", err)
-	}
-}
-
-func completionHandler(w http.ResponseWriter, req *http.Request) {
-	// post request
-	body := req.Body
-	// get body as io.reader
-	// pass it to the /completion
-	go sendMsgToLLM(body)
-out:
-	for {
-		select {
-		case chunk := <-chunkChan:
-			fmt.Print(chunk)
-			if _, err := w.Write([]byte(chunk)); err != nil {
-				logger.Warn("failed to write chunk", "value", chunk)
-				continue
-			}
-		case <-streamDone:
-			break out
-		}
-	}
-}
-
-func modelHandler(w http.ResponseWriter, req *http.Request) {
-	llmModel := fetchLCPModelName()
-	payload, err := json.Marshal(llmModel)
-	if err != nil {
-		logger.Error("model handler", "error", err)
-		// return err
-		return
-	}
-	if _, err := w.Write(payload); err != nil {
-		logger.Error("model handler", "error", err)
-	}
-}

session.go

@@ -131,13 +131,18 @@ func loadOldChatOrGetNew() []models.RoleMsg {
 	chat, err := store.GetLastChat()
 	if err != nil {
 		logger.Warn("failed to load history chat", "error", err)
+		maxID, err := store.ChatGetMaxID()
+		if err != nil {
+			logger.Error("failed to fetch max chat id", "error", err)
+		}
+		maxID++
 		chat := &models.Chat{
-			ID:        0,
+			ID:        maxID,
 			CreatedAt: time.Now(),
 			UpdatedAt: time.Now(),
 			Agent:     cfg.AssistantRole,
 		}
-		chat.Name = fmt.Sprintf("%s_%v", chat.Agent, chat.CreatedAt.Unix())
+		chat.Name = fmt.Sprintf("%s_%v", chat.Agent, chat.ID)
 		activeChatName = chat.Name
 		chatMap[chat.Name] = chat
 		return defaultStarter
@@ -149,10 +154,6 @@ func loadOldChatOrGetNew() []models.RoleMsg {
 		chatMap[chat.Name] = chat
 		return defaultStarter
 	}
-	// if chat.Name == "" {
-	// 	logger.Warn("empty chat name", "id", chat.ID)
-	// 	chat.Name = fmt.Sprintf("%s_%v", chat.Agent, chat.CreatedAt.Unix())
-	// }
 	chatMap[chat.Name] = chat
 	activeChatName = chat.Name
 	cfg.AssistantRole = chat.Agent

storage/storage.go

@@ -103,7 +103,6 @@ func NewProviderSQL(dbPath string, logger *slog.Logger) FullRepo {
 		return nil
 	}
 	p := ProviderSQL{db: db, logger: logger}
-
 	p.Migrate()
 	return p
 }

storage/vector.go

@@ -73,12 +73,9 @@ func (p ProviderSQL) WriteVector(row *models.VectorRow) error {
 	if err != nil {
 		return err
 	}
-
 	serializedEmbeddings := SerializeVector(row.Embeddings)
-
 	query := fmt.Sprintf("INSERT INTO %s(embeddings, slug, raw_text, filename) VALUES (?, ?, ?, ?)", tableName)
 	_, err = p.db.Exec(query, serializedEmbeddings, row.Slug, row.RawText, row.FileName)
-
 	return err
 }
 
@@ -87,27 +84,22 @@ func (p ProviderSQL) SearchClosest(q []float32) ([]models.VectorRow, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	querySQL := "SELECT embeddings, slug, raw_text, filename FROM " + tableName
 	rows, err := p.db.Query(querySQL)
 	if err != nil {
 		return nil, err
 	}
 	defer rows.Close()
-
 	type SearchResult struct {
 		vector   models.VectorRow
 		distance float32
 	}
-
 	var topResults []SearchResult
-
 	for rows.Next() {
 		var (
-			embeddingsBlob []byte
+			embeddingsBlob          []byte
 			slug, rawText, fileName string
 		)
-
 		if err := rows.Scan(&embeddingsBlob, &slug, &rawText, &fileName); err != nil {
 			continue
 		}
@@ -152,7 +144,6 @@ func (p ProviderSQL) SearchClosest(q []float32) ([]models.VectorRow, error) {
 		result.vector.Distance = result.distance
 		results[i] = result.vector
 	}
-
 	return results, nil
 }
 
@@ -161,18 +152,15 @@ func cosineSimilarity(a, b []float32) float32 {
 	if len(a) != len(b) {
 		return 0.0
 	}
-
 	var dotProduct, normA, normB float32
 	for i := 0; i < len(a); i++ {
 		dotProduct += a[i] * b[i]
 		normA += a[i] * a[i]
 		normB += b[i] * b[i]
 	}
-
 	if normA == 0 || normB == 0 {
 		return 0.0
 	}
-
 	return dotProduct / (sqrt(normA) * sqrt(normB))
 }
 
@@ -229,13 +217,11 @@ func (p ProviderSQL) ListFiles() ([]string, error) {
 			}
 		}
 	}
-
 	return allFiles, nil
 }
 
 func (p ProviderSQL) RemoveEmbByFileName(filename string) error {
 	var errors []string
-
 	tableNames := []string{
 		"embeddings_384", "embeddings_768", "embeddings_1024", "embeddings_1536",
 		"embeddings_2048", "embeddings_3072", "embeddings_4096", "embeddings_5120",
@@ -246,10 +232,8 @@ func (p ProviderSQL) RemoveEmbByFileName(filename string) error {
 			errors = append(errors, err.Error())
 		}
 	}
-
 	if len(errors) > 0 {
 		return fmt.Errorf("errors occurred: %v", errors)
 	}
-
 	return nil
 }

sysprompts/alice_bob_carl.json

@@ -0,0 +1,7 @@
+{
+  "sys_prompt": "This is a chat between Alice, Bob and Carl. Normally all message are public (seen by everyone). But characters also able to make messages intended to specific targets using '@' tag. Usually tag is provided inside of out of character clause: (ooc: @charname@), but will be parsed if put anywhere in the message.\nTO SEND A PRIVATE MESSAGE:\n- Include a recipient tag in this exact format: @CharacterName@\n- The tag can be anywhere in your message\n- Example: \"(ooc: @Bob@) Don't tell others this secret.\"\n- For immersion sake it is better if private messages are given in context of whispering, passing notes, or being alone in some space: Alice: (ooc: @Carl@) *leans closer to Carl and whispers* \"I forgot to turn off the car, could you watch my bag for a cuple of minutes?\"\n- Only the sender and tagged recipients will see that message.\nRECEIVING MESSAGES:\n- You only see messages where you are the sender OR you are tagged in the recipient tag\n- Public messages (without tags) are seen by everyone.\nEXAMPLE FORMAT:\nAlice: \"Public message everyone sees\"\nAlice: (ooc: @Bob@)\n\"Private message only for Bob\"\n(if Diana joins the conversation, and Alice wants to exclude her) Alice: (ooc: @Bob,Carl@; Diana is not trustworthy)\n*Grabs Bob and Carl, and pulls them away* \"Listen boys, let's meet this friday again!\"\nWHEN TO USE:\n- Most of the time public messages (no tag) are the best choice. Private messages (with tag) are mostly for the passing secrets or information that is described or infered as private.\n- Game of 20 questions. Guys are putting paper sickers on the forehead with names written on them. So in this case only person who gets the sticker put on them does not see the writting on it.\nBob: *Puts sticker with 'JACK THE RIPPER' written on it, on Alices forehead* (ooc: @Carl).\nCarl: \"Alright, we're ready.\"\nAlice: \"Good. So, am I a fictional character or a real one?\"",
+  "role": "Alice",
+  "filepath": "sysprompts/alice_bob_carl.json",
+  "chars": ["Alice", "Bob", "Carl"],
+  "first_msg": "\"Hey guys! Want to play Alias like game? I'll tell Bob a word and he needs to describe that word so Carl can guess what it was?\""
+}

sysprompts/coding_assistant.json

@@ -0,0 +1,6 @@
+{
+  "sys_prompt": "You are an expert software engineering assistant. Your goal is to help users with coding tasks, debugging, refactoring, and software development.\n\n## Core Principles\n1. **Security First**: Never expose secrets, keys, or credentials. Never commit sensitive data.\n2. **No Git Actions**: You can READ git info (status, log, diff) for context, but NEVER perform git actions (commit, add, push, checkout, reset, rm, etc.). Let the user handle all git operations.\n3. **Explore Before Execute**: Always understand the codebase structure before making changes.\n4. **Follow Conventions**: Match existing code style, patterns, and frameworks used in the project.\n5. **Be Concise**: Minimize output tokens while maintaining quality. Avoid unnecessary explanations.\n6. **Ask First**: When uncertain about intent, ask the user. Don't assume.\n\n## Workflow for Complex Tasks\nFor multi-step tasks, ALWAYS use the todo system to track progress:\n\n1. **Create Todo List**: At the start of complex tasks, use `todo_create` to break down work into actionable items.\n2. **Update Progress**: Mark items as `in_progress` when working on them, and `completed` when done.\n3. **Check Status**: Use `todo_read` to review your progress.\n\nExample workflow:\n- User: \"Add user authentication to this app\"\n- You: Create todos: [\"Analyze existing auth structure\", \"Check frameworks in use\", \"Implement auth middleware\", \"Add login endpoints\", \"Test implementation\"]\n\n## Task Execution Flow\n\n### Phase 1: Exploration (Always First)\n- Use `file_list` to understand directory structure (path defaults to FilePickerDir if not specified)\n- Use `file_read` to examine relevant files (paths are relative to FilePickerDir unless starting with `/`)\n- Use `execute_command` with `grep`/`find` to search for patterns\n- Check README, Makefile, package.json, or similar for build/test commands\n- Identify: frameworks, conventions, testing approach, lint/typecheck commands\n- **Git reads allowed**: You may use `git status`, `git log`, `git diff` for context, but only to inform your work\n- **Path handling**: Relative paths resolve against FilePickerDir; absolute paths (starting with `/`) bypass it\n\n### Phase 2: Planning\n- For complex tasks: create todo items\n- Identify files that need modification\n- Plan your approach following existing patterns\n\n### Phase 3: Implementation\n- Make changes using appropriate file tools\n- Prefer `file_write` for new files, `file_read` then edit for existing files\n- Follow existing code style exactly\n- Use existing libraries and utilities\n\n### Phase 4: Verification\n- Run tests if available (check for test scripts in README/Makefile)\n- Run linting/type checking commands\n- Verify changes work as expected\n\n### Phase 5: Completion\n- Update todos to `completed`\n- Provide concise summary of changes\n- Reference specific file paths and line numbers when relevant\n- **DO NOT commit changes** - inform user what was done so they can review and commit themselves\n\n## Command Execution\n- Use `execute_command` with a single string containing command and arguments (e.g., `go run main.go`, `ls -la`, `cd /tmp`)\n- Use `cd /path` to change the working directory for file operations",
+  "role": "CodingAssistant",
+  "filepath": "sysprompts/coding_assistant.json",
+  "first_msg": "Hello! I'm your coding assistant. Give me a specific task and I'll get started. For complex work, I'll track progress with todos."
+}

tables.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"image"
 	"os"
 	"path"
 	"strings"
@@ -39,7 +40,7 @@ func makeChatTable(chatMap map[string]models.Chat) *tview.Table {
 	// Add header row (row 0)
 	for c := 0; c < cols; c++ {
 		color := tcell.ColorWhite
-		headerText := ""
+		var headerText string
 		switch c {
 		case 0:
 			headerText = "Chat Name"
@@ -128,7 +129,7 @@ func makeChatTable(chatMap map[string]models.Chat) *tview.Table {
 				return
 			}
 			chatBody.Messages = history
-			textView.SetText(chatToText(cfg.ShowSys))
+			textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
 			activeChatName = selectedChat
 			pages.RemovePage(historyPage)
 			return
@@ -151,7 +152,7 @@ func makeChatTable(chatMap map[string]models.Chat) *tview.Table {
 			}
 			// load last chat
 			chatBody.Messages = loadOldChatOrGetNew()
-			textView.SetText(chatToText(cfg.ShowSys))
+			textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
 			pages.RemovePage(historyPage)
 			return
 		case "update card":
@@ -184,7 +185,7 @@ func makeChatTable(chatMap map[string]models.Chat) *tview.Table {
 		case "move sysprompt onto 1st msg":
 			chatBody.Messages[1].Content = chatBody.Messages[0].Content + chatBody.Messages[1].Content
 			chatBody.Messages[0].Content = rpDefenitionSysMsg
-			textView.SetText(chatToText(cfg.ShowSys))
+			textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
 			activeChatName = selectedChat
 			pages.RemovePage(historyPage)
 			return
@@ -215,8 +216,8 @@ func makeChatTable(chatMap map[string]models.Chat) *tview.Table {
 			}
 			// Update sysMap with fresh card data
 			sysMap[agentName] = newCard
-			applyCharCard(newCard)
-			startNewChat()
+			// fetching sysprompt and first message anew from the card
+			startNewChat(false)
 			pages.RemovePage(historyPage)
 			return
 		default:
@@ -235,9 +236,59 @@ func makeChatTable(chatMap map[string]models.Chat) *tview.Table {
 }
 
 // nolint:unused
-func makeRAGTable(fileList []string) *tview.Flex {
-	actions := []string{"load", "delete"}
-	rows, cols := len(fileList), len(actions)+1
+func formatSize(size int64) string {
+	units := []string{"B", "KB", "MB", "GB", "TB"}
+	i := 0
+	s := float64(size)
+	for s >= 1024 && i < len(units)-1 {
+		s /= 1024
+		i++
+	}
+	return fmt.Sprintf("%.1f%s", s, units[i])
+}
+
+type ragFileInfo struct {
+	name     string
+	inRAGDir bool
+	isLoaded bool
+	fullPath string
+}
+
+func makeRAGTable(fileList []string, loadedFiles []string) *tview.Flex {
+	// Build set of loaded files for quick lookup
+	loadedSet := make(map[string]bool)
+	for _, f := range loadedFiles {
+		loadedSet[f] = true
+	}
+
+	// Build merged list: files from ragdir + orphaned files from DB
+	ragFiles := make([]ragFileInfo, 0, len(fileList)+len(loadedFiles))
+	seen := make(map[string]bool)
+
+	// Add files from ragdir
+	for _, f := range fileList {
+		ragFiles = append(ragFiles, ragFileInfo{
+			name:     f,
+			inRAGDir: true,
+			isLoaded: loadedSet[f],
+			fullPath: path.Join(cfg.RAGDir, f),
+		})
+		seen[f] = true
+	}
+
+	// Add orphaned files (in DB but not in ragdir)
+	for _, f := range loadedFiles {
+		if !seen[f] {
+			ragFiles = append(ragFiles, ragFileInfo{
+				name:     f,
+				inRAGDir: false,
+				isLoaded: true,
+				fullPath: "",
+			})
+		}
+	}
+	rows := len(ragFiles)
+	cols := 4 // File Name | Preview | Action | Delete
 	fileTable := tview.NewTable().
 		SetBorders(true)
 	longStatusView := tview.NewTextView()
@@ -251,40 +302,92 @@ func makeRAGTable(fileList []string) *tview.Flex {
 		AddItem(fileTable, 0, 60, true)
 	// Add the exit option as the first row (row 0)
 	fileTable.SetCell(0, 0,
-		tview.NewTableCell("Exit RAG manager").
+		tview.NewTableCell("File Name").
 			SetTextColor(tcell.ColorWhite).
 			SetAlign(tview.AlignCenter).
 			SetSelectable(false))
 	fileTable.SetCell(0, 1,
-		tview.NewTableCell("(Close without action)").
-			SetTextColor(tcell.ColorGray).
+		tview.NewTableCell("Preview").
+			SetTextColor(tcell.ColorWhite).
 			SetAlign(tview.AlignCenter).
 			SetSelectable(false))
 	fileTable.SetCell(0, 2,
-		tview.NewTableCell("exit").
-			SetTextColor(tcell.ColorGray).
-			SetAlign(tview.AlignCenter))
+		tview.NewTableCell("Load/Unload").
+			SetTextColor(tcell.ColorWhite).
+			SetAlign(tview.AlignCenter).
+			SetSelectable(false))
+	fileTable.SetCell(0, 3,
+		tview.NewTableCell("Delete").
+			SetTextColor(tcell.ColorWhite).
+			SetAlign(tview.AlignCenter).
+			SetSelectable(false))
 	// Add the file rows starting from row 1
 	for r := 0; r < rows; r++ {
+		f := ragFiles[r]
 		for c := 0; c < cols; c++ {
 			color := tcell.ColorWhite
-			if c < 1 {
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell(fileList[r]).
+			switch c {
+			case 0:
+				displayName := f.name
+				if !f.inRAGDir {
+					displayName = f.name + " (orphaned)"
+				}
+				fileTable.SetCell(r+1, c,
+					tview.NewTableCell(displayName).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter).
 						SetSelectable(false))
-			} else if c == 1 { // Action description column - not selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell("(Action)").
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter).
-						SetSelectable(false))
-			} else { // Action button column - selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell(actions[c-1]).
+			case 1:
+				if !f.inRAGDir {
+					// Orphaned file - no preview available
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell("not in ragdir").
+							SetTextColor(tcell.ColorYellow).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				} else if fi, err := os.Stat(f.fullPath); err == nil {
+					size := fi.Size()
+					modTime := fi.ModTime()
+					preview := fmt.Sprintf("%s | %s", formatSize(size), modTime.Format("2006-01-02 15:04"))
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell(preview).
+							SetTextColor(color).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				} else {
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell("error").
+							SetTextColor(color).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				}
+			case 2:
+				actionText := "load"
+				if f.isLoaded {
+					actionText = "unload"
+				}
+				if !f.inRAGDir {
+					// Orphaned file - can only unload
+					actionText = "unload"
+				}
+				fileTable.SetCell(r+1, c,
+					tview.NewTableCell(actionText).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter))
+			case 3:
+				if !f.inRAGDir {
+					// Orphaned file - cannot delete from ragdir (not there)
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell("-").
+							SetTextColor(tcell.ColorDarkGray).
+							SetAlign(tview.AlignCenter).
+							SetSelectable(false))
+				} else {
+					fileTable.SetCell(r+1, c,
+						tview.NewTableCell("delete").
+							SetTextColor(color).
+							SetAlign(tview.AlignCenter))
+				}
 			}
 		}
 	}
@@ -316,7 +419,7 @@ func makeRAGTable(fileList []string) *tview.Flex {
 	}()
 	fileTable.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') || key == tcell.KeyCtrlX {
@@ -333,30 +436,58 @@ func makeRAGTable(fileList []string) *tview.Flex {
 		}
 		// defer pages.RemovePage(RAGPage)
 		tc := fileTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		fileTable.SetSelectable(false, false)
 		// Check if the selected row is the exit row (row 0) - do this first to avoid index issues
 		if row == 0 {
 			pages.RemovePage(RAGPage)
 			return
 		}
-		// For file rows, get the filename (row index - 1 because of the exit row at index 0)
-		fpath := fileList[row-1] // -1 to account for the exit row at index 0
-		// notification := fmt.Sprintf("chat: %s; action: %s", fpath, tc.Text)
+		// For file rows, get the file info (row index - 1 because of the exit row at index 0)
+		f := ragFiles[row-1]
+		// Handle "-" case (orphaned file with no delete option)
+		if tc.Text == "-" {
+			pages.RemovePage(RAGPage)
+			return
+		}
 		switch tc.Text {
 		case "load":
-			fpath = path.Join(cfg.RAGDir, fpath)
+			fpath := path.Join(cfg.RAGDir, f.name)
 			longStatusView.SetText("clicked load")
 			go func() {
 				if err := ragger.LoadRAG(fpath); err != nil {
 					logger.Error("failed to embed file", "chat", fpath, "error", err)
 					_ = notifyUser("RAG", "failed to embed file; error: "+err.Error())
-					errCh <- err
-					// pages.RemovePage(RAGPage)
+					app.QueueUpdate(func() {
+						pages.RemovePage(RAGPage)
+					})
 					return
 				}
+				_ = notifyUser("RAG", "file loaded successfully")
+				app.QueueUpdate(func() {
+					pages.RemovePage(RAGPage)
+				})
+			}()
+			return
+		case "unload":
+			longStatusView.SetText("clicked unload")
+			go func() {
+				if err := ragger.RemoveFile(f.name); err != nil {
+					logger.Error("failed to unload file from RAG", "filename", f.name, "error", err)
+					_ = notifyUser("RAG", "failed to unload file; error: "+err.Error())
+					app.QueueUpdate(func() {
+						pages.RemovePage(RAGPage)
+					})
+					return
+				}
+				_ = notifyUser("RAG", "file unloaded successfully")
+				app.QueueUpdate(func() {
+					pages.RemovePage(RAGPage)
+				})
 			}()
 			return
 		case "delete":
-			fpath = path.Join(cfg.RAGDir, fpath)
+			fpath := path.Join(cfg.RAGDir, f.name)
 			if err := os.Remove(fpath); err != nil {
 				logger.Error("failed to delete file", "filename", fpath, "error", err)
 				return
@@ -381,113 +512,6 @@ func makeRAGTable(fileList []string) *tview.Flex {
 	return ragflex
 }
 
-func makeLoadedRAGTable(fileList []string) *tview.Flex {
-	actions := []string{"delete"}
-	rows, cols := len(fileList), len(actions)+1
-	// Add 1 extra row for the "exit" option at the top
-	fileTable := tview.NewTable().
-		SetBorders(true)
-	longStatusView := tview.NewTextView()
-	longStatusView.SetText("Loaded RAG files list")
-	longStatusView.SetBorder(true).SetTitle("status")
-	longStatusView.SetChangedFunc(func() {
-		app.Draw()
-	})
-	ragflex := tview.NewFlex().SetDirection(tview.FlexRow).
-		AddItem(longStatusView, 0, 10, false).
-		AddItem(fileTable, 0, 60, true)
-	// Add the exit option as the first row (row 0)
-	fileTable.SetCell(0, 0,
-		tview.NewTableCell("Exit Loaded Files manager").
-			SetTextColor(tcell.ColorWhite).
-			SetAlign(tview.AlignCenter).
-			SetSelectable(false))
-	fileTable.SetCell(0, 1,
-		tview.NewTableCell("(Close without action)").
-			SetTextColor(tcell.ColorGray).
-			SetAlign(tview.AlignCenter).
-			SetSelectable(false))
-	fileTable.SetCell(0, 2,
-		tview.NewTableCell("exit").
-			SetTextColor(tcell.ColorGray).
-			SetAlign(tview.AlignCenter))
-	// Add the file rows starting from row 1
-	for r := 0; r < rows; r++ {
-		for c := 0; c < cols; c++ {
-			color := tcell.ColorWhite
-			if c < 1 {
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell(fileList[r]).
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter).
-						SetSelectable(false))
-			} else if c == 1 { // Action description column - not selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell("(Action)").
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter).
-						SetSelectable(false))
-			} else { // Action button column - selectable
-				fileTable.SetCell(r+1, c, // +1 to account for the exit row at index 0
-					tview.NewTableCell(actions[c-1]).
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter))
-			}
-		}
-	}
-	fileTable.Select(0, 0).
-		SetFixed(1, 1).
-		SetSelectable(true, false).
-		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
-		SetDoneFunc(func(key tcell.Key) {
-			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') || key == tcell.KeyCtrlX {
-				pages.RemovePage(RAGLoadedPage)
-				return
-			}
-		}).SetSelectedFunc(func(row int, column int) {
-		// If user selects a non-actionable column (0 or 1), move to first action column (2)
-		if column <= 1 {
-			if fileTable.GetColumnCount() > 2 {
-				fileTable.Select(row, 2) // Select first action column
-			}
-			return
-		}
-		tc := fileTable.GetCell(row, column)
-		// Check if the selected row is the exit row (row 0) - do this first to avoid index issues
-		if row == 0 {
-			pages.RemovePage(RAGLoadedPage)
-			return
-		}
-		// For file rows, get the filename (row index - 1 because of the exit row at index 0)
-		fpath := fileList[row-1] // -1 to account for the exit row at index 0
-		switch tc.Text {
-		case "delete":
-			if err := ragger.RemoveFile(fpath); err != nil {
-				logger.Error("failed to delete file from RAG", "filename", fpath, "error", err)
-				longStatusView.SetText(fmt.Sprintf("Error deleting file: %v", err))
-				return
-			}
-			if err := notifyUser("RAG file deleted", fpath+" was deleted from RAG system"); err != nil {
-				logger.Error("failed to send notification", "error", err)
-			}
-			longStatusView.SetText(fpath + " was deleted from RAG system")
-			return
-		default:
-			pages.RemovePage(RAGLoadedPage)
-			return
-		}
-	})
-	// Add input capture to the flex container to handle 'x' key for closing
-	ragflex.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
-		if event.Key() == tcell.KeyRune && event.Rune() == 'x' {
-			pages.RemovePage(RAGLoadedPage)
-			return nil
-		}
-		return event
-	})
-	return ragflex
-}
-
 func makeAgentTable(agentList []string) *tview.Table {
 	actions := []string{"filepath", "load"}
 	rows, cols := len(agentList), len(actions)+1
@@ -496,13 +520,14 @@ func makeAgentTable(agentList []string) *tview.Table {
 	for r := 0; r < rows; r++ {
 		for c := 0; c < cols; c++ {
 			color := tcell.ColorWhite
-			if c < 1 {
+			switch c {
+			case 0:
 				chatActTable.SetCell(r, c,
 					tview.NewTableCell(agentList[r]).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter).
 						SetSelectable(false))
-			} else if c == 1 {
+			case 1:
 				if actions[c-1] == "filepath" {
 					cc, ok := sysMap[agentList[r]]
 					if !ok {
@@ -519,7 +544,7 @@ func makeAgentTable(agentList []string) *tview.Table {
 					tview.NewTableCell(actions[c-1]).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter))
-			} else {
+			default:
 				chatActTable.SetCell(r, c,
 					tview.NewTableCell(actions[c-1]).
 						SetTextColor(color).
@@ -529,7 +554,7 @@ func makeAgentTable(agentList []string) *tview.Table {
 	}
 	chatActTable.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') {
@@ -545,17 +570,19 @@ func makeAgentTable(agentList []string) *tview.Table {
 			return
 		}
 		tc := chatActTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		chatActTable.SetSelectable(false, false)
 		selected := agentList[row]
 		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
 		switch tc.Text {
 		case "load":
-			if ok := charToStart(selected); !ok {
+			if ok := charToStart(selected, true); !ok {
 				logger.Warn("no such sys msg", "name", selected)
 				pages.RemovePage(agentPage)
 				return
 			}
 			// replace textview
-			textView.SetText(chatToText(cfg.ShowSys))
+			textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
 			colorText()
 			updateStatusLine()
 			// sysModal.ClearButtons()
@@ -609,13 +636,14 @@ func makeCodeBlockTable(codeBlocks []string) *tview.Table {
 			if len(codeBlocks[r]) < 30 {
 				previewLen = len(codeBlocks[r])
 			}
-			if c < 1 {
+			switch {
+			case c < 1:
 				table.SetCell(r, c,
 					tview.NewTableCell(codeBlocks[r][:previewLen]).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter).
 						SetSelectable(false))
-			} else {
+			default:
 				table.SetCell(r, c,
 					tview.NewTableCell(actions[c-1]).
 						SetTextColor(color).
@@ -625,7 +653,7 @@ func makeCodeBlockTable(codeBlocks []string) *tview.Table {
 	}
 	table.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') {
@@ -641,6 +669,8 @@ func makeCodeBlockTable(codeBlocks []string) *tview.Table {
 			return
 		}
 		tc := table.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		table.SetSelectable(false, false)
 		selected := codeBlocks[row]
 		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
 		switch tc.Text {
@@ -680,13 +710,14 @@ func makeImportChatTable(filenames []string) *tview.Table {
 	for r := 0; r < rows; r++ {
 		for c := 0; c < cols; c++ {
 			color := tcell.ColorWhite
-			if c < 1 {
+			switch {
+			case c < 1:
 				chatActTable.SetCell(r, c,
 					tview.NewTableCell(filenames[r]).
 						SetTextColor(color).
 						SetAlign(tview.AlignCenter).
 						SetSelectable(false))
-			} else {
+			default:
 				chatActTable.SetCell(r, c,
 					tview.NewTableCell(actions[c-1]).
 						SetTextColor(color).
@@ -696,7 +727,7 @@ func makeImportChatTable(filenames []string) *tview.Table {
 	}
 	chatActTable.Select(0, 0).
 		SetFixed(1, 1).
-		SetSelectable(true, false).
+		SetSelectable(true, true).
 		SetSelectedStyle(tcell.StyleDefault.Background(tcell.ColorGray).Foreground(tcell.ColorWhite)).
 		SetDoneFunc(func(key tcell.Key) {
 			if key == tcell.KeyEsc || key == tcell.KeyF1 || key == tcell.Key('x') {
@@ -712,6 +743,8 @@ func makeImportChatTable(filenames []string) *tview.Table {
 			return
 		}
 		tc := chatActTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		chatActTable.SetSelectable(false, false)
 		selected := filenames[row]
 		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
 		switch tc.Text {
@@ -724,7 +757,7 @@ func makeImportChatTable(filenames []string) *tview.Table {
 			colorText()
 			updateStatusLine()
 			// redraw the text in text area
-			textView.SetText(chatToText(cfg.ShowSys))
+			textView.SetText(chatToText(chatBody.Messages, cfg.ShowSys))
 			pages.RemovePage(historyPage)
 			app.SetFocus(textArea)
 			return
@@ -783,17 +816,19 @@ func makeFilePicker() *tview.Flex {
 	var selectedFile string
 	// Track currently displayed directory (changes as user navigates)
 	currentDisplayDir := startDir
+	// --- NEW: search state ---
+	searching := false
+	searchQuery := ""
+	searchInputMode := false
 	// Helper function to check if a file has an allowed extension from config
 	hasAllowedExtension := func(filename string) bool {
-		// If no allowed extensions are specified in config, allow all files
 		if cfg.FilePickerExts == "" {
 			return true
 		}
-		// Split the allowed extensions from the config string
 		allowedExts := strings.Split(cfg.FilePickerExts, ",")
 		lowerFilename := strings.ToLower(strings.TrimSpace(filename))
 		for _, ext := range allowedExts {
-			ext = strings.TrimSpace(ext) // Remove any whitespace around the extension
+			ext = strings.TrimSpace(ext)
 			if ext != "" && strings.HasSuffix(lowerFilename, "."+ext) {
 				return true
 			}
@@ -813,21 +848,39 @@ func makeFilePicker() *tview.Flex {
 	}
 	// Create UI elements
 	listView := tview.NewList()
-	listView.SetBorder(true).SetTitle("Files & Directories").SetTitleAlign(tview.AlignLeft)
+	listView.SetBorder(true).
+		SetTitle("Files & Directories [s: set FilePickerDir]. Current base dir: " + cfg.FilePickerDir).
+		SetTitleAlign(tview.AlignLeft)
 	// Status view for selected file information
 	statusView := tview.NewTextView()
 	statusView.SetBorder(true).SetTitle("Selected File").SetTitleAlign(tview.AlignLeft)
 	statusView.SetTextColor(tcell.ColorYellow)
-	// Layout - only include list view and status view
+	// Image preview pane
+	var imgPreview *tview.Image
+	if cfg.ImagePreview {
+		imgPreview = tview.NewImage()
+		imgPreview.SetBorder(true).SetTitle("Preview").SetTitleAlign(tview.AlignLeft)
+	}
+	// Horizontal flex for list + preview
+	var hFlex *tview.Flex
+	if cfg.ImagePreview && imgPreview != nil {
+		hFlex = tview.NewFlex().SetDirection(tview.FlexColumn).
+			AddItem(listView, 0, 3, true).
+			AddItem(imgPreview, 0, 2, false)
+	} else {
+		hFlex = tview.NewFlex().SetDirection(tview.FlexColumn).
+			AddItem(listView, 0, 1, true)
+	}
+	// Main vertical flex
 	flex := tview.NewFlex().SetDirection(tview.FlexRow)
-	flex.AddItem(listView, 0, 3, true)
+	flex.AddItem(hFlex, 0, 3, true)
 	flex.AddItem(statusView, 3, 0, false)
-	// Refresh the file list
-	var refreshList func(string)
-	refreshList = func(dir string) {
+	// Refresh the file list – now accepts a filter string
+	var refreshList func(string, string)
+	refreshList = func(dir string, filter string) {
 		listView.Clear()
 		// Update the current display directory
-		currentDisplayDir = dir // Update the current display directory
+		currentDisplayDir = dir
 		// Add exit option at the top
 		listView.AddItem("Exit file picker [gray](Close without selecting)[-]", "", 'x', func() {
 			pages.RemovePage(filePickerPage)
@@ -835,13 +888,16 @@ func makeFilePicker() *tview.Flex {
 		// Add parent directory (..) if not at root
 		if dir != "/" {
 			parentDir := path.Dir(dir)
-			// Special handling for edge cases - only return if we're truly at a system root
-			// For Unix-like systems, path.Dir("/") returns "/" which would cause parentDir == dir
-			if parentDir == dir && dir == "/" {
-				// We're at the root ("/") and trying to go up, just don't add the parent item
-			} else {
+			// For Unix-like systems, avoid infinite loop when at root
+			if parentDir != dir {
 				listView.AddItem("../ [gray](Parent Directory)[-]", "", 'p', func() {
-					refreshList(parentDir)
+					// Clear search on navigation
+					searching = false
+					searchQuery = ""
+					if cfg.ImagePreview {
+						imgPreview.SetImage(nil)
+					}
+					refreshList(parentDir, "")
 					dirStack = append(dirStack, parentDir)
 					currentStackPos = len(dirStack) - 1
 				})
@@ -853,95 +909,257 @@ func makeFilePicker() *tview.Flex {
 			statusView.SetText("Error reading directory: " + err.Error())
 			return
 		}
-		// Add directories and files to the list
+		// Helper to check if an item passes the filter
+		matchesFilter := func(name string) bool {
+			if filter == "" {
+				return true
+			}
+			return strings.Contains(strings.ToLower(name), strings.ToLower(filter))
+		}
+		// Add directories
 		for _, file := range files {
 			name := file.Name()
-			// Skip hidden files and directories (those starting with a dot)
 			if strings.HasPrefix(name, ".") {
 				continue
 			}
-			if file.IsDir() {
-				// Capture the directory name for the closure to avoid loop variable issues
+			if file.IsDir() && matchesFilter(name) {
 				dirName := name
 				listView.AddItem(dirName+"/ [gray](Directory)[-]", "", 0, func() {
+					// Clear search on navigation
+					searching = false
+					searchQuery = ""
+					if cfg.ImagePreview {
+						imgPreview.SetImage(nil)
+					}
 					newDir := path.Join(dir, dirName)
-					refreshList(newDir)
+					refreshList(newDir, "")
 					dirStack = append(dirStack, newDir)
 					currentStackPos = len(dirStack) - 1
 					statusView.SetText("Current: " + newDir)
 				})
-			} else {
-				// Only show files that have allowed extensions (from config)
-				if hasAllowedExtension(name) {
-					// Capture the file name for the closure to avoid loop variable issues
-					fileName := name
-					fullFilePath := path.Join(dir, fileName)
-					listView.AddItem(fileName+" [gray](File)[-]", "", 0, func() {
-						selectedFile = fullFilePath
-						statusView.SetText("Selected: " + selectedFile)
-						// Check if the file is an image
-						if isImageFile(fileName) {
-							// For image files, offer to attach to the next LLM message
-							statusView.SetText("Selected image: " + selectedFile)
-						} else {
-							// For non-image files, display as before
-							statusView.SetText("Selected: " + selectedFile)
-						}
-					})
-				}
 			}
 		}
-		statusView.SetText("Current: " + dir)
+		// Add files with allowed extensions
+		for _, file := range files {
+			name := file.Name()
+			if strings.HasPrefix(name, ".") || file.IsDir() {
+				continue
+			}
+			if hasAllowedExtension(name) && matchesFilter(name) {
+				fileName := name
+				fullFilePath := path.Join(dir, fileName)
+				listView.AddItem(fileName+" [gray](File)[-]", "", 0, func() {
+					selectedFile = fullFilePath
+					statusView.SetText("Selected: " + selectedFile)
+					if isImageFile(fileName) {
+						statusView.SetText("Selected image: " + selectedFile)
+					}
+				})
+			}
+		}
+		// Update status line based on search state
+		switch {
+		case searching:
+			statusView.SetText("Search: " + searchQuery + "_")
+		case searchQuery != "":
+			statusView.SetText("Current: " + dir + " (filter: " + searchQuery + ")")
+		default:
+			statusView.SetText("Current: " + dir)
+		}
 	}
 	// Initialize the file list
-	refreshList(startDir)
+	refreshList(startDir, "")
+	// Update image preview when selection changes (unchanged)
+	if cfg.ImagePreview && imgPreview != nil {
+		listView.SetChangedFunc(func(index int, mainText, secondaryText string, rune rune) {
+			itemText, _ := listView.GetItemText(index)
+			if strings.HasPrefix(itemText, "Exit file picker") || strings.HasPrefix(itemText, "../") {
+				imgPreview.SetImage(nil)
+				return
+			}
+			actualItemName := itemText
+			if bracketPos := strings.Index(itemText, " ["); bracketPos != -1 {
+				actualItemName = itemText[:bracketPos]
+			}
+			if strings.HasSuffix(actualItemName, "/") {
+				imgPreview.SetImage(nil)
+				return
+			}
+			if !isImageFile(actualItemName) {
+				imgPreview.SetImage(nil)
+				return
+			}
+			filePath := path.Join(currentDisplayDir, actualItemName)
+			file, err := os.Open(filePath)
+			if err != nil {
+				imgPreview.SetImage(nil)
+				return
+			}
+			defer file.Close()
+			img, _, err := image.Decode(file)
+			if err != nil {
+				imgPreview.SetImage(nil)
+				return
+			}
+			imgPreview.SetImage(img)
+		})
+	}
 	// Set up keyboard navigation
 	flex.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		// --- Handle search mode ---
+		if searching {
+			switch event.Key() {
+			case tcell.KeyEsc:
+				// Exit search, clear filter
+				searching = false
+				searchInputMode = false
+				searchQuery = ""
+				refreshList(currentDisplayDir, "")
+				return nil
+			case tcell.KeyBackspace, tcell.KeyBackspace2:
+				if len(searchQuery) > 0 {
+					searchQuery = searchQuery[:len(searchQuery)-1]
+					refreshList(currentDisplayDir, searchQuery)
+				}
+				return nil
+			case tcell.KeyEnter:
+				// Exit search input mode and let normal processing handle selection
+				searchInputMode = false
+				// Get the currently highlighted item in the list
+				itemIndex := listView.GetCurrentItem()
+				if itemIndex >= 0 && itemIndex < listView.GetItemCount() {
+					itemText, _ := listView.GetItemText(itemIndex)
+					// Check for the exit option first
+					if strings.HasPrefix(itemText, "Exit file picker") {
+						pages.RemovePage(filePickerPage)
+						return nil
+					}
+					// Extract the actual filename/directory name by removing the type info
+					actualItemName := itemText
+					if bracketPos := strings.Index(itemText, " ["); bracketPos != -1 {
+						actualItemName = itemText[:bracketPos]
+					}
+					// Check if it's a directory (ends with /)
+					if strings.HasSuffix(actualItemName, "/") {
+						var targetDir string
+						if strings.HasPrefix(actualItemName, "../") {
+							// Parent directory
+							targetDir = path.Dir(currentDisplayDir)
+							if targetDir == currentDisplayDir && currentDisplayDir == "/" {
+								return nil
+							}
+						} else {
+							// Regular subdirectory
+							dirName := strings.TrimSuffix(actualItemName, "/")
+							targetDir = path.Join(currentDisplayDir, dirName)
+						}
+						// Navigate – clear search
+						if cfg.ImagePreview && imgPreview != nil {
+							imgPreview.SetImage(nil)
+						}
+						searching = false
+						searchInputMode = false
+						searchQuery = ""
+						refreshList(targetDir, "")
+						dirStack = append(dirStack, targetDir)
+						currentStackPos = len(dirStack) - 1
+						statusView.SetText("Current: " + targetDir)
+						return nil
+					} else {
+						// It's a file
+						filePath := path.Join(currentDisplayDir, actualItemName)
+						if info, err := os.Stat(filePath); err == nil && !info.IsDir() {
+							if isImageFile(actualItemName) {
+								SetImageAttachment(filePath)
+								statusView.SetText("Image attached: " + filePath + " (will be sent with next message)")
+								pages.RemovePage(filePickerPage)
+							} else {
+								textArea.SetText(filePath, true)
+								app.SetFocus(textArea)
+								pages.RemovePage(filePickerPage)
+							}
+						}
+						return nil
+					}
+				}
+				return nil
+			case tcell.KeyRune:
+				r := event.Rune()
+				if searchInputMode && r != 0 {
+					searchQuery += string(r)
+					refreshList(currentDisplayDir, searchQuery)
+					return nil
+				}
+				// If not in search input mode, pass through for navigation
+				return event
+			default:
+				// Exit search input mode but keep filter active for navigation
+				searchInputMode = false
+				// Pass all other keys (arrows, etc.) to normal processing
+				return event
+			}
+		}
+		// --- Not searching ---
 		switch event.Key() {
 		case tcell.KeyEsc:
 			pages.RemovePage(filePickerPage)
 			return nil
 		case tcell.KeyBackspace2: // Backspace to go to parent directory
+			if cfg.ImagePreview && imgPreview != nil {
+				imgPreview.SetImage(nil)
+			}
 			if currentStackPos > 0 {
 				currentStackPos--
 				prevDir := dirStack[currentStackPos]
-				refreshList(prevDir)
-				// Trim the stack to current position to avoid deep history
+				// Clear search when navigating with backspace
+				searching = false
+				searchQuery = ""
+				refreshList(prevDir, "")
+				// Trim the stack to current position
 				dirStack = dirStack[:currentStackPos+1]
 			}
 			return nil
+		case tcell.KeyRune:
+			if event.Rune() == '/' {
+				// Enter search mode
+				searching = true
+				searchInputMode = true
+				searchQuery = ""
+				refreshList(currentDisplayDir, "")
+				return nil
+			}
+			if event.Rune() == 's' {
+				// Set FilePickerDir to current directory
+				// Get the actual directory path
+				cfg.FilePickerDir = currentDisplayDir
+				listView.SetTitle("Files & Directories [s: set FilePickerDir]. Current base dir: " + cfg.FilePickerDir)
+				// pages.RemovePage(filePickerPage)
+				return nil
+			}
 		case tcell.KeyEnter:
 			// Get the currently highlighted item in the list
 			itemIndex := listView.GetCurrentItem()
 			if itemIndex >= 0 && itemIndex < listView.GetItemCount() {
-				// We need to get the text of the currently selected item to determine if it's a directory
-				// Since we can't directly get the item text, we'll keep track of items differently
-				// Let's improve the approach by tracking the currently selected item
 				itemText, _ := listView.GetItemText(itemIndex)
 				logger.Info("choosing dir", "itemText", itemText)
-				// Check for the exit option first (should be the first item)
+				// Check for the exit option first
 				if strings.HasPrefix(itemText, "Exit file picker") {
 					pages.RemovePage(filePickerPage)
 					return nil
 				}
-				// Extract the actual filename/directory name by removing the type info in brackets
-				// Format is "name [gray](type)[-]"
+				// Extract the actual filename/directory name by removing the type info
 				actualItemName := itemText
 				if bracketPos := strings.Index(itemText, " ["); bracketPos != -1 {
 					actualItemName = itemText[:bracketPos]
 				}
 				// Check if it's a directory (ends with /)
 				if strings.HasSuffix(actualItemName, "/") {
-					// This is a directory, we need to get the full path
-					// Since the item text ends with "/" and represents a directory
 					var targetDir string
 					if strings.HasPrefix(actualItemName, "../") {
-						// Parent directory - need to go up from current directory
+						// Parent directory
 						targetDir = path.Dir(currentDisplayDir)
-						// Avoid going above root - if parent is same as current and it's system root
 						if targetDir == currentDisplayDir && currentDisplayDir == "/" {
-							// We're at root, don't navigate
-							logger.Warn("went to root", "dir", targetDir)
+							logger.Warn("at root, cannot go up")
 							return nil
 						}
 					} else {
@@ -949,24 +1167,23 @@ func makeFilePicker() *tview.Flex {
 						dirName := strings.TrimSuffix(actualItemName, "/")
 						targetDir = path.Join(currentDisplayDir, dirName)
 					}
-					// Navigate to the selected directory
-					logger.Info("going to the dir", "dir", targetDir)
-					refreshList(targetDir)
+					// Navigate – clear search
+					logger.Info("going to dir", "dir", targetDir)
+					if cfg.ImagePreview && imgPreview != nil {
+						imgPreview.SetImage(nil)
+					}
+					searching = false
+					searchQuery = ""
+					refreshList(targetDir, "")
 					dirStack = append(dirStack, targetDir)
 					currentStackPos = len(dirStack) - 1
 					statusView.SetText("Current: " + targetDir)
 					return nil
 				} else {
-					// It's a file - construct the full path from current directory and the actual item name
-					// We can't rely only on the selectedFile variable since Enter key might be pressed
-					// without having clicked the file first
+					// It's a file
 					filePath := path.Join(currentDisplayDir, actualItemName)
-					// Verify it's actually a file (not just lacking a directory suffix)
 					if info, err := os.Stat(filePath); err == nil && !info.IsDir() {
-						// Check if the file is an image
 						if isImageFile(actualItemName) {
-							// For image files, set it as an attachment for the next LLM message
-							// Use the version without UI updates to avoid hangs in event handlers
 							logger.Info("setting image", "file", actualItemName)
 							SetImageAttachment(filePath)
 							logger.Info("after setting image", "file", actualItemName)
@@ -975,7 +1192,6 @@ func makeFilePicker() *tview.Flex {
 							pages.RemovePage(filePickerPage)
 							logger.Info("after update drawn", "file", actualItemName)
 						} else {
-							// For non-image files, update the text area with file path
 							textArea.SetText(filePath, true)
 							app.SetFocus(textArea)
 							pages.RemovePage(filePickerPage)

tools.go

@@ -9,12 +9,15 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"gf-lt/rag"
+
 	"github.com/GrailFinder/searchagent/searcher"
 )
 
@@ -57,9 +60,9 @@ Your current tools:
 "when_to_use": "when asked to search the web for information; returns clean summary without html,css and other web elements; limit is optional (default 3)"
 },
 {
-"name":"websearch_raw",
+"name":"rag_search",
 "args": ["query", "limit"],
-"when_to_use": "when asked to search the web for information; returns raw data as is without processing; limit is optional (default 3)"
+"when_to_use": "when asked to search the local document database for information; performs query refinement, semantic search, reranking, and synthesis; returns clean summary with sources; limit is optional (default 3)"
 },
 {
 "name":"read_url",
@@ -92,6 +95,11 @@ Your current tools:
 "when_to_use": "when asked to append content to a file; use sed to edit content"
 },
 {
+"name":"file_edit",
+"args": ["path", "oldString", "newString", "lineNumber"],
+"when_to_use": "when you need to make targeted changes to a specific section of a file without rewriting the entire file; lineNumber is optional - if provided, only edits that specific line; if not provided, replaces all occurrences of oldString"
+},
+{
 "name":"file_delete",
 "args": ["path"],
 "when_to_use": "when asked to delete a file"
@@ -114,7 +122,7 @@ Your current tools:
 {
 "name":"execute_command",
 "args": ["command", "args"],
-"when_to_use": "when asked to execute a system command; args is optional; allowed commands: grep, sed, awk, find, cat, head, tail, sort, uniq, wc, ls, echo, cut, tr, cp, mv, rm, mkdir, rmdir, pwd, df, free, ps, top, du, whoami, date, uname"
+"when_to_use": "when asked to execute a system command; args is optional; allowed commands: grep, sed, awk, find, cat, head, tail, sort, uniq, wc, ls, echo, cut, tr, cp, mv, rm, mkdir, rmdir, pwd, df, free, ps, top, du, whoami, date, uname, go"
 }
 ]
 </tools>
@@ -145,6 +153,7 @@ under the topic: Adam's number is stored:
 After that you are free to respond to the user.
 `
 	webSearchSysPrompt = `Summarize the web search results, extracting key information and presenting a concise answer. Provide sources and URLs where relevant.`
+	ragSearchSysPrompt = `Synthesize the document search results, extracting key information and presenting a concise answer. Provide sources and document IDs where relevant.`
 	readURLSysPrompt   = `Extract and summarize the content from the webpage. Provide key information, main points, and any relevant details.`
 	summarySysPrompt   = `Please provide a concise summary of the following conversation. Focus on key points, decisions, and actions. Provide only the summary, no additional commentary.`
 	basicCard          = &models.CharCard{
@@ -169,6 +178,9 @@ func init() {
 		panic("failed to init seachagent; error: " + err.Error())
 	}
 	WebSearcher = sa
+	if err := rag.Init(cfg, logger, store); err != nil {
+		logger.Warn("failed to init rag; rag_search tool will not be available", "error", err)
+	}
 }
 
 // getWebAgentClient returns a singleton AgentClient for web agents.
@@ -195,6 +207,8 @@ func getWebAgentClient() *agent.AgentClient {
 func registerWebAgents() {
 	webAgentsOnce.Do(func() {
 		client := getWebAgentClient()
+		// Register rag_search agent
+		agent.Register("rag_search", agent.NewWebAgentB(client, ragSearchSysPrompt))
 		// Register websearch agent
 		agent.Register("websearch", agent.NewWebAgentB(client, webSearchSysPrompt))
 		// Register read_url agent
@@ -238,6 +252,45 @@ func websearch(args map[string]string) []byte {
 	return data
 }
 
+// rag search (searches local document database)
+func ragsearch(args map[string]string) []byte {
+	query, ok := args["query"]
+	if !ok || query == "" {
+		msg := "query not provided to rag_search tool"
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	limitS, ok := args["limit"]
+	if !ok || limitS == "" {
+		limitS = "3"
+	}
+	limit, err := strconv.Atoi(limitS)
+	if err != nil || limit == 0 {
+		logger.Warn("ragsearch limit; passed bad value; setting to default (3)",
+			"limit_arg", limitS, "error", err)
+		limit = 3
+	}
+	ragInstance := rag.GetInstance()
+	if ragInstance == nil {
+		msg := "rag not initialized; rag_search tool is not available"
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	results, err := ragInstance.Search(query, limit)
+	if err != nil {
+		msg := "rag search failed; error: " + err.Error()
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	data, err := json.Marshal(results)
+	if err != nil {
+		msg := "failed to marshal rag search result; error: " + err.Error()
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	return data
+}
+
 // web search raw (returns raw data without processing)
 func websearchRaw(args map[string]string) []byte {
 	// make http request return bytes
@@ -368,7 +421,6 @@ func recallTopics(args map[string]string) []byte {
 }
 
 // File Manipulation Tools
-
 func fileCreate(args map[string]string) []byte {
 	path, ok := args["path"]
 	if !ok || path == "" {
@@ -376,18 +428,16 @@ func fileCreate(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
+	path = resolvePath(path)
 	content, ok := args["content"]
 	if !ok {
 		content = ""
 	}
-
 	if err := writeStringToFile(path, content); err != nil {
 		msg := "failed to create file; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	msg := "file created successfully at " + path
 	return []byte(msg)
 }
@@ -399,14 +449,13 @@ func fileRead(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
+	path = resolvePath(path)
 	content, err := readStringFromFile(path)
 	if err != nil {
 		msg := "failed to read file; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	result := map[string]string{
 		"content": content,
 		"path":    path,
@@ -417,7 +466,6 @@ func fileRead(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	return jsonResult
 }
 
@@ -428,6 +476,7 @@ func fileWrite(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
+	path = resolvePath(path)
 	content, ok := args["content"]
 	if !ok {
 		content = ""
@@ -448,6 +497,7 @@ func fileWriteAppend(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
+	path = resolvePath(path)
 	content, ok := args["content"]
 	if !ok {
 		content = ""
@@ -461,6 +511,77 @@ func fileWriteAppend(args map[string]string) []byte {
 	return []byte(msg)
 }
 
+func fileEdit(args map[string]string) []byte {
+	path, ok := args["path"]
+	if !ok || path == "" {
+		msg := "path not provided to file_edit tool"
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	path = resolvePath(path)
+	oldString, ok := args["oldString"]
+	if !ok || oldString == "" {
+		msg := "oldString not provided to file_edit tool"
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	newString, ok := args["newString"]
+	if !ok {
+		newString = ""
+	}
+	lineNumberStr, hasLineNumber := args["lineNumber"]
+	// Read file content
+	content, err := os.ReadFile(path)
+	if err != nil {
+		msg := "failed to read file: " + err.Error()
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	fileContent := string(content)
+	var replacementCount int
+	if hasLineNumber && lineNumberStr != "" {
+		// Line-number based edit
+		lineNum, err := strconv.Atoi(lineNumberStr)
+		if err != nil {
+			msg := "invalid lineNumber: must be a valid integer"
+			logger.Error(msg)
+			return []byte(msg)
+		}
+		lines := strings.Split(fileContent, "\n")
+		if lineNum < 1 || lineNum > len(lines) {
+			msg := fmt.Sprintf("lineNumber %d out of range (file has %d lines)", lineNum, len(lines))
+			logger.Error(msg)
+			return []byte(msg)
+		}
+		// Find oldString in the specific line
+		targetLine := lines[lineNum-1]
+		if !strings.Contains(targetLine, oldString) {
+			msg := fmt.Sprintf("oldString not found on line %d", lineNum)
+			logger.Error(msg)
+			return []byte(msg)
+		}
+		lines[lineNum-1] = strings.Replace(targetLine, oldString, newString, 1)
+		replacementCount = 1
+		fileContent = strings.Join(lines, "\n")
+	} else {
+		// Replace all occurrences
+		if !strings.Contains(fileContent, oldString) {
+			msg := "oldString not found in file"
+			logger.Error(msg)
+			return []byte(msg)
+		}
+		fileContent = strings.ReplaceAll(fileContent, oldString, newString)
+		replacementCount = strings.Count(fileContent, newString)
+	}
+	if err := os.WriteFile(path, []byte(fileContent), 0644); err != nil {
+		msg := "failed to write file: " + err.Error()
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	msg := fmt.Sprintf("file edited successfully at %s (%d replacement(s))", path, replacementCount)
+	return []byte(msg)
+}
+
 func fileDelete(args map[string]string) []byte {
 	path, ok := args["path"]
 	if !ok || path == "" {
@@ -468,13 +589,12 @@ func fileDelete(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
+	path = resolvePath(path)
 	if err := removeFile(path); err != nil {
 		msg := "failed to delete file; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	msg := "file deleted successfully at " + path
 	return []byte(msg)
 }
@@ -486,20 +606,19 @@ func fileMove(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
+	src = resolvePath(src)
 	dst, ok := args["dst"]
 	if !ok || dst == "" {
 		msg := "destination path not provided to file_move tool"
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
+	dst = resolvePath(dst)
 	if err := moveFile(src, dst); err != nil {
 		msg := "failed to move file; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	msg := fmt.Sprintf("file moved successfully from %s to %s", src, dst)
 	return []byte(msg)
 }
@@ -511,20 +630,19 @@ func fileCopy(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
+	src = resolvePath(src)
 	dst, ok := args["dst"]
 	if !ok || dst == "" {
 		msg := "destination path not provided to file_copy tool"
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
+	dst = resolvePath(dst)
 	if err := copyFile(src, dst); err != nil {
 		msg := "failed to copy file; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	msg := fmt.Sprintf("file copied successfully from %s to %s", src, dst)
 	return []byte(msg)
 }
@@ -534,14 +652,13 @@ func fileList(args map[string]string) []byte {
 	if !ok || path == "" {
 		path = "." // default to current directory
 	}
-
+	path = resolvePath(path)
 	files, err := listDirectory(path)
 	if err != nil {
 		msg := "failed to list directory; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	result := map[string]interface{}{
 		"directory": path,
 		"files":     files,
@@ -552,11 +669,16 @@ func fileList(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	return jsonResult
 }
 
 // Helper functions for file operations
+func resolvePath(p string) string {
+	if filepath.IsAbs(p) {
+		return p
+	}
+	return filepath.Join(cfg.FilePickerDir, p)
+}
 
 func readStringFromFile(filename string) (string, error) {
 	data, err := os.ReadFile(filename)
@@ -576,7 +698,6 @@ func appendStringToFile(filename string, data string) error {
 		return err
 	}
 	defer file.Close()
-
 	_, err = file.WriteString(data)
 	return err
 }
@@ -600,13 +721,11 @@ func copyFile(src, dst string) error {
 		return err
 	}
 	defer srcFile.Close()
-
 	dstFile, err := os.Create(dst)
 	if err != nil {
 		return err
 	}
 	defer dstFile.Close()
-
 	_, err = io.Copy(dstFile, srcFile)
 	return err
 }
@@ -625,7 +744,6 @@ func listDirectory(path string) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	var files []string
 	for _, entry := range entries {
 		if entry.IsDir() {
@@ -634,79 +752,120 @@ func listDirectory(path string) ([]string, error) {
 			files = append(files, entry.Name())
 		}
 	}
-
 	return files, nil
 }
 
 // Command Execution Tool
-
 func executeCommand(args map[string]string) []byte {
-	command, ok := args["command"]
-	if !ok || command == "" {
+	commandStr := args["command"]
+	if commandStr == "" {
 		msg := "command not provided to execute_command tool"
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
-	if !isCommandAllowed(command) {
+	// Handle commands passed as single string with spaces (e.g., "go run main.go" or "cd /tmp")
+	// Split into base command and arguments
+	parts := strings.Fields(commandStr)
+	if len(parts) == 0 {
+		msg := "command not provided to execute_command tool"
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	command := parts[0]
+	cmdArgs := parts[1:]
+	if !isCommandAllowed(command, cmdArgs...) {
 		msg := fmt.Sprintf("command '%s' is not allowed", command)
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
-	// Get arguments - handle both single arg and multiple args
-	var cmdArgs []string
-	if args["args"] != "" {
-		// If args is provided as a single string, split by spaces
-		cmdArgs = strings.Fields(args["args"])
-	} else {
-		// If individual args are provided, collect them
-		argNum := 1
-		for {
-			argKey := fmt.Sprintf("arg%d", argNum)
-			if argValue, exists := args[argKey]; exists && argValue != "" {
-				cmdArgs = append(cmdArgs, argValue)
-			} else {
-				break
-			}
-			argNum++
-		}
+	// Special handling for cd command - update FilePickerDir
+	if command == "cd" {
+		return handleCdCommand(cmdArgs)
 	}
-
 	// Execute with timeout for safety
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	cmd := exec.CommandContext(ctx, command, cmdArgs...)
-
+	cmd.Dir = cfg.FilePickerDir
 	output, err := cmd.CombinedOutput()
 	if err != nil {
 		msg := fmt.Sprintf("command '%s' failed; error: %v; output: %s", command, err, string(output))
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	// Check if output is empty and return success message
 	if len(output) == 0 {
-		successMsg := fmt.Sprintf("command '%s %s' executed successfully and exited with code 0", command, strings.Join(cmdArgs, " "))
+		successMsg := fmt.Sprintf("command '%s' executed successfully and exited with code 0", commandStr)
 		return []byte(successMsg)
 	}
-
 	return output
 }
 
-// Helper functions for command execution
+// handleCdCommand handles the cd command to update FilePickerDir
+func handleCdCommand(args []string) []byte {
+	var targetDir string
+	if len(args) == 0 {
+		// cd with no args goes to home directory
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			msg := "cd: cannot determine home directory: " + err.Error()
+			logger.Error(msg)
+			return []byte(msg)
+		}
+		targetDir = homeDir
+	} else {
+		targetDir = args[0]
+	}
 
+	// Resolve relative paths against current FilePickerDir
+	if !filepath.IsAbs(targetDir) {
+		targetDir = filepath.Join(cfg.FilePickerDir, targetDir)
+	}
+
+	// Verify the directory exists
+	info, err := os.Stat(targetDir)
+	if err != nil {
+		msg := "cd: " + targetDir + ": " + err.Error()
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	if !info.IsDir() {
+		msg := "cd: " + targetDir + ": not a directory"
+		logger.Error(msg)
+		return []byte(msg)
+	}
+
+	// Update FilePickerDir
+	absDir, err := filepath.Abs(targetDir)
+	if err != nil {
+		msg := "cd: failed to resolve path: " + err.Error()
+		logger.Error(msg)
+		return []byte(msg)
+	}
+	cfg.FilePickerDir = absDir
+	msg := "FilePickerDir changed to: " + absDir
+	return []byte(msg)
+}
+
+// Helper functions for command execution
 // Todo structure
 type TodoItem struct {
 	ID     string `json:"id"`
 	Task   string `json:"task"`
 	Status string `json:"status"` // "pending", "in_progress", "completed"
 }
-
 type TodoList struct {
 	Items []TodoItem `json:"items"`
 }
 
+func (t TodoList) ToString() string {
+	sb := strings.Builder{}
+	for i := range t.Items {
+		fmt.Fprintf(&sb, "\n[%s] %s. %s\n", t.Items[i].Status, t.Items[i].ID, t.Items[i].Task)
+	}
+	return sb.String()
+}
+
 // Global todo list storage
 var globalTodoList = TodoList{
 	Items: []TodoItem{},
@@ -720,69 +879,34 @@ func todoCreate(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	// Generate simple ID
 	id := fmt.Sprintf("todo_%d", len(globalTodoList.Items)+1)
-
 	newItem := TodoItem{
 		ID:     id,
 		Task:   task,
 		Status: "pending",
 	}
-
 	globalTodoList.Items = append(globalTodoList.Items, newItem)
-
 	result := map[string]string{
 		"message": "todo created successfully",
 		"id":      id,
 		"task":    task,
 		"status":  "pending",
+		"todos":   globalTodoList.ToString(),
 	}
-
 	jsonResult, err := json.Marshal(result)
 	if err != nil {
 		msg := "failed to marshal result; error: " + err.Error()
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	return jsonResult
 }
 
 func todoRead(args map[string]string) []byte {
-	id, ok := args["id"]
-	if ok && id != "" {
-		// Find specific todo by ID
-		for _, item := range globalTodoList.Items {
-			if item.ID == id {
-				result := map[string]interface{}{
-					"todo": item,
-				}
-				jsonResult, err := json.Marshal(result)
-				if err != nil {
-					msg := "failed to marshal result; error: " + err.Error()
-					logger.Error(msg)
-					return []byte(msg)
-				}
-				return jsonResult
-			}
-		}
-		// ID not found
-		result := map[string]string{
-			"error": fmt.Sprintf("todo with id %s not found", id),
-		}
-		jsonResult, err := json.Marshal(result)
-		if err != nil {
-			msg := "failed to marshal result; error: " + err.Error()
-			logger.Error(msg)
-			return []byte(msg)
-		}
-		return jsonResult
-	}
-
 	// Return all todos if no ID specified
 	result := map[string]interface{}{
-		"todos": globalTodoList.Items,
+		"todos": globalTodoList.ToString(),
 	}
 	jsonResult, err := json.Marshal(result)
 	if err != nil {
@@ -790,7 +914,6 @@ func todoRead(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	return jsonResult
 }
 
@@ -801,16 +924,13 @@ func todoUpdate(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	task, taskOk := args["task"]
 	status, statusOk := args["status"]
-
 	if !taskOk && !statusOk {
 		msg := "neither task nor status provided to todo_update tool"
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	// Find and update the todo
 	for i, item := range globalTodoList.Items {
 		if item.ID == id {
@@ -834,23 +954,20 @@ func todoUpdate(args map[string]string) []byte {
 					return jsonResult
 				}
 			}
-
 			result := map[string]string{
 				"message": "todo updated successfully",
 				"id":      id,
+				"todos":   globalTodoList.ToString(),
 			}
-
 			jsonResult, err := json.Marshal(result)
 			if err != nil {
 				msg := "failed to marshal result; error: " + err.Error()
 				logger.Error(msg)
 				return []byte(msg)
 			}
-
 			return jsonResult
 		}
 	}
-
 	// ID not found
 	result := map[string]string{
 		"error": fmt.Sprintf("todo with id %s not found", id),
@@ -871,29 +988,25 @@ func todoDelete(args map[string]string) []byte {
 		logger.Error(msg)
 		return []byte(msg)
 	}
-
 	// Find and remove the todo
 	for i, item := range globalTodoList.Items {
 		if item.ID == id {
 			// Remove item from slice
 			globalTodoList.Items = append(globalTodoList.Items[:i], globalTodoList.Items[i+1:]...)
-
 			result := map[string]string{
 				"message": "todo deleted successfully",
 				"id":      id,
+				"todos":   globalTodoList.ToString(),
 			}
-
 			jsonResult, err := json.Marshal(result)
 			if err != nil {
 				msg := "failed to marshal result; error: " + err.Error()
 				logger.Error(msg)
 				return []byte(msg)
 			}
-
 			return jsonResult
 		}
 	}
-
 	// ID not found
 	result := map[string]string{
 		"error": fmt.Sprintf("todo with id %s not found", id),
@@ -907,8 +1020,21 @@ func todoDelete(args map[string]string) []byte {
 	return jsonResult
 }
 
-func isCommandAllowed(command string) bool {
+var gitReadSubcommands = map[string]bool{
+	"status":    true,
+	"log":       true,
+	"diff":      true,
+	"show":      true,
+	"branch":    true,
+	"reflog":    true,
+	"rev-parse": true,
+	"shortlog":  true,
+	"describe":  true,
+}
+
+func isCommandAllowed(command string, args ...string) bool {
 	allowedCommands := map[string]bool{
+		"cd":     true,
 		"grep":   true,
 		"sed":    true,
 		"awk":    true,
@@ -937,8 +1063,20 @@ func isCommandAllowed(command string) bool {
 		"whoami": true,
 		"date":   true,
 		"uname":  true,
+		"git":    true,
+		"go":     true,
 	}
-	return allowedCommands[command]
+	// Allow all go subcommands (go run, go mod tidy, go test, etc.)
+	if strings.HasPrefix(command, "go ") && allowedCommands["go"] {
+		return true
+	}
+	if command == "git" && len(args) > 0 {
+		return gitReadSubcommands[args[0]]
+	}
+	if !allowedCommands[command] {
+		return false
+	}
+	return true
 }
 
 func summarizeChat(args map[string]string) []byte {
@@ -946,7 +1084,7 @@ func summarizeChat(args map[string]string) []byte {
 		return []byte("No chat history to summarize.")
 	}
 	// Format chat history for the agent
-	chatText := chatToText(true) // include system and tool messages
+	chatText := chatToText(chatBody.Messages, true) // include system and tool messages
 	return []byte(chatText)
 }
 
@@ -956,6 +1094,7 @@ var fnMap = map[string]fnSig{
 	"recall":            recall,
 	"recall_topics":     recallTopics,
 	"memorise":          memorise,
+	"rag_search":        ragsearch,
 	"websearch":         websearch,
 	"websearch_raw":     websearchRaw,
 	"read_url":          readURL,
@@ -964,6 +1103,7 @@ var fnMap = map[string]fnSig{
 	"file_read":         fileRead,
 	"file_write":        fileWrite,
 	"file_write_append": fileWriteAppend,
+	"file_edit":         fileEdit,
 	"file_delete":       fileDelete,
 	"file_move":         fileMove,
 	"file_copy":         fileCopy,
@@ -992,6 +1132,28 @@ func callToolWithAgent(name string, args map[string]string) []byte {
 
 // openai style def
 var baseTools = []models.Tool{
+	// rag_search
+	models.Tool{
+		Type: "function",
+		Function: models.ToolFunc{
+			Name:        "rag_search",
+			Description: "Search local document database given query, limit of sources (default 3). Performs query refinement, semantic search, reranking, and synthesis.",
+			Parameters: models.ToolFuncParams{
+				Type:     "object",
+				Required: []string{"query", "limit"},
+				Properties: map[string]models.ToolArgProps{
+					"query": models.ToolArgProps{
+						Type:        "string",
+						Description: "search query",
+					},
+					"limit": models.ToolArgProps{
+						Type:        "string",
+						Description: "limit of the document results",
+					},
+				},
+			},
+		},
+	},
 	// websearch
 	models.Tool{
 		Type: "function",
@@ -1125,7 +1287,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_create
 	models.Tool{
 		Type: "function",
@@ -1148,7 +1309,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_read
 	models.Tool{
 		Type: "function",
@@ -1167,7 +1327,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_write
 	models.Tool{
 		Type: "function",
@@ -1190,7 +1349,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_write_append
 	models.Tool{
 		Type: "function",
@@ -1213,7 +1371,36 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
+	// file_edit
+	models.Tool{
+		Type: "function",
+		Function: models.ToolFunc{
+			Name:        "file_edit",
+			Description: "Edit a specific section of a file by replacing oldString with newString. Use for targeted changes without rewriting the entire file.",
+			Parameters: models.ToolFuncParams{
+				Type:     "object",
+				Required: []string{"path", "oldString", "newString"},
+				Properties: map[string]models.ToolArgProps{
+					"path": models.ToolArgProps{
+						Type:        "string",
+						Description: "path of the file to edit",
+					},
+					"oldString": models.ToolArgProps{
+						Type:        "string",
+						Description: "the exact string to find and replace",
+					},
+					"newString": models.ToolArgProps{
+						Type:        "string",
+						Description: "the string to replace oldString with",
+					},
+					"lineNumber": models.ToolArgProps{
+						Type:        "string",
+						Description: "optional line number (1-indexed) to edit - if provided, only that line is edited",
+					},
+				},
+			},
+		},
+	},
 	// file_delete
 	models.Tool{
 		Type: "function",
@@ -1232,7 +1419,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_move
 	models.Tool{
 		Type: "function",
@@ -1255,7 +1441,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_copy
 	models.Tool{
 		Type: "function",
@@ -1278,7 +1463,6 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// file_list
 	models.Tool{
 		Type: "function",
@@ -1297,24 +1481,19 @@ var baseTools = []models.Tool{
 			},
 		},
 	},
-
 	// execute_command
 	models.Tool{
 		Type: "function",
 		Function: models.ToolFunc{
 			Name:        "execute_command",
-			Description: "Execute a shell command safely. Use when you need to run system commands like grep sed awk find cat head tail sort uniq wc ls echo cut tr cp mv rm mkdir rmdir pwd df free ps top du whoami date uname",
+			Description: "Execute a shell command safely. Use when you need to run system commands like cd grep sed awk find cat head tail sort uniq wc ls echo cut tr cp mv rm mkdir rmdir pwd df free ps top du whoami date uname go git. Git is allowed for read-only operations: status, log, diff, show, branch, reflog, rev-parse, shortlog, describe. Use 'cd /path' to change working directory.",
 			Parameters: models.ToolFuncParams{
 				Type:     "object",
 				Required: []string{"command"},
 				Properties: map[string]models.ToolArgProps{
 					"command": models.ToolArgProps{
 						Type:        "string",
-						Description: "command to execute (only commands from whitelist are allowed: grep sed awk find cat head tail sort uniq wc ls echo cut tr cp mv rm mkdir rmdir pwd df free ps top du whoami date uname",
-					},
-					"args": models.ToolArgProps{
-						Type:        "string",
-						Description: "command arguments as a single string (e.g., '-la {path}')",
+						Description: "command to execute with arguments (e.g., 'go run main.go', 'ls -la /tmp', 'cd /home/user'). Use a single string; arguments should be space-separated after the command.",
 					},
 				},
 			},
@@ -1376,7 +1555,7 @@ var baseTools = []models.Tool{
 					},
 					"status": models.ToolArgProps{
 						Type:        "string",
-						Description: "new status for the todo: pending, in_progress, or completed (optional)",
+						Description: "new status: pending, in_progress, or completed (optional)",
 					},
 				},
 			},