Fix: RAG updates

README.md

@@ -23,6 +23,7 @@
 - export whole chat into a json file; +
 - directory with sys prompts (charcards png & json); +
 - colourschemes, colours or markdown of quotes and styles; (partially done) +
+- source file name to group by rag vectors; +
 - change temp, min-p and other params from tui;
 - fullscreen textarea option (bothersome to implement);
 - consider adding use /completion of llamacpp, since openai endpoint clearly has template|format issues;
@@ -34,6 +35,7 @@
 - delete chat option;
 - server mode: no tui but api calls with the func calling, rag, other middleware;
 - boolean flag to use/not use tools. I see it as a msg from a tool to an llm "Hey, it might be good idea to use me!";
+- RAG file loading status/progress;
 
 ### FIX:
 - bot responding (or hanging) blocks everything; +
@@ -53,5 +55,8 @@
 - normal case regen omits assistant icon; +
 - user icon (and role?) from config is not used; +
 - message editing broke ( runtime error: index out of range [-1]); +
+- RAG: encode multiple sentences (~5-10) to embeddings a piece. +
+- number of sentences in a batch should depend on number of words there. +
 - F1 can load any chat, by loading chat of other agent it does not switch agents, if that chat is continued, it will rewrite agent in db; (either allow only chats from current agent OR switch agent on chat loading);
 - after chat is deleted: load undeleted chat;
+- edit mode remove extra \n, but it should not be there in a first place. after edit no styles

bot.go

@@ -138,17 +138,12 @@ func chatRagUse(qText string) (string, error) {
 			logger.Error("failed to get embs", "error", err, "index", i, "question", q)
 			continue
 		}
-		// e := &models.EmbeddingResp{
-		// 	Embedding: emb,
-		// }
-		// vecs, err := ragger.SearchEmb(e)
 		vecs, err := store.SearchClosest(emb)
 		if err != nil {
 			logger.Error("failed to query embs", "error", err, "index", i, "question", q)
 			continue
 		}
 		respVecs = append(respVecs, vecs...)
-		// logger.Info("returned vector from query search", "question", q, "vec", vec)
 	}
 	// get raw text
 	resps := []string{}

models/db.go

@@ -43,4 +43,5 @@ type VectorRow struct {
 	Slug       string    `db:"slug" json:"slug"`
 	RawText    string    `db:"raw_text" json:"raw_text"`
 	Distance   float32   `db:"distance" json:"distance"`
+	FileName   string    `db:"filename" json:"filename"`
 }

rag/main.go

@@ -11,6 +11,8 @@ import (
 	"log/slog"
 	"net/http"
 	"os"
+	"path"
+	"strings"
 
 	"github.com/neurosnap/sentences/english"
 )
@@ -29,6 +31,10 @@ func New(l *slog.Logger, s storage.FullRepo, cfg *config.Config) *RAG {
 	}
 }
 
+func wordCounter(sentence string) int {
+	return len(strings.Split(sentence, " "))
+}
+
 func (r *RAG) LoadRAG(fpath string) error {
 	data, err := os.ReadFile(fpath)
 	if err != nil {
@@ -53,6 +59,9 @@ func (r *RAG) LoadRAG(fpath string) error {
 		batchSize = 200
 		maxChSize = 1000
 		//
+		// psize     = 3
+		wordLimit = 80
+		//
 		left     = 0
 		right    = batchSize
 		batchCh  = make(chan map[int][]string, maxChSize)
@@ -60,23 +69,40 @@ func (r *RAG) LoadRAG(fpath string) error {
 		errCh    = make(chan error, 1)
 		doneCh   = make(chan bool, 1)
 	)
-	if len(sents) < batchSize {
-		batchSize = len(sents)
+	// group sentences
+	paragraphs := []string{}
+	par := strings.Builder{}
+	for i := 0; i < len(sents); i++ {
+		par.WriteString(sents[i])
+		if wordCounter(par.String()) > wordLimit {
+			paragraphs = append(paragraphs, par.String())
+			par.Reset()
+		}
+	}
+	// for i := 0; i < len(sents); i += psize {
+	// 	if len(sents) < i+psize {
+	// 		paragraphs = append(paragraphs, strings.Join(sents[i:], " "))
+	// 		break
+	// 	}
+	// 	paragraphs = append(paragraphs, strings.Join(sents[i:i+psize], " "))
+	// }
+	if len(paragraphs) < batchSize {
+		batchSize = len(paragraphs)
 	}
 	// fill input channel
 	ctn := 0
 	for {
-		if right > len(sents) {
-			batchCh <- map[int][]string{left: sents[left:]}
+		if right > len(paragraphs) {
+			batchCh <- map[int][]string{left: paragraphs[left:]}
 			break
 		}
-		batchCh <- map[int][]string{left: sents[left:right]}
+		batchCh <- map[int][]string{left: paragraphs[left:right]}
 		left, right = right, right+batchSize
 		ctn++
 	}
 	r.logger.Info("finished batching", "batches#", len(batchCh))
 	for w := 0; w < workers; w++ {
-		go r.batchToVectorHFAsync(len(sents), batchCh, vectorCh, errCh, doneCh)
+		go r.batchToVectorHFAsync(len(paragraphs), batchCh, vectorCh, errCh, doneCh, path.Base(fpath))
 	}
 	// write to db
 	return r.writeVectors(vectorCh, doneCh)
@@ -102,20 +128,17 @@ func (r *RAG) writeVectors(vectorCh <-chan []models.VectorRow, doneCh <-chan boo
 }
 
 func (r *RAG) batchToVectorHFAsync(limit int, inputCh <-chan map[int][]string,
-	vectorCh chan<- []models.VectorRow, errCh chan error, doneCh chan bool) {
+	vectorCh chan<- []models.VectorRow, errCh chan error, doneCh chan bool, filename string) {
 	r.logger.Info("to vector batches", "batches#", len(inputCh))
 	for {
 		select {
 		case linesMap := <-inputCh:
-			// r.logger.Info("batch from ch")
 			for leftI, v := range linesMap {
-				// r.logger.Info("fetching", "index", leftI)
-				r.fecthEmbHF(v, errCh, vectorCh, fmt.Sprintf("test_%d", leftI))
+				r.fecthEmbHF(v, errCh, vectorCh, fmt.Sprintf("%s_%d", filename, leftI), filename)
 				if leftI+200 >= limit { // last batch
 					doneCh <- true
 					return
 				}
-				// r.logger.Info("done feitching", "index", leftI)
 			}
 		case <-doneCh:
 			r.logger.Info("got done")
@@ -129,7 +152,7 @@ func (r *RAG) batchToVectorHFAsync(limit int, inputCh <-chan map[int][]string,
 	}
 }
 
-func (r *RAG) fecthEmbHF(lines []string, errCh chan error, vectorCh chan<- []models.VectorRow, slug string) {
+func (r *RAG) fecthEmbHF(lines []string, errCh chan error, vectorCh chan<- []models.VectorRow, slug, filename string) {
 	payload, err := json.Marshal(
 		map[string]any{"inputs": lines, "options": map[string]bool{"wait_for_model": true}},
 	)
@@ -138,13 +161,14 @@ func (r *RAG) fecthEmbHF(lines []string, errCh chan error, vectorCh chan<- []mod
 		errCh <- err
 		return
 	}
+	// nolint
 	req, err := http.NewRequest("POST", r.cfg.EmbedURL, bytes.NewReader(payload))
 	if err != nil {
 		r.logger.Error("failed to create new req", "err:", err.Error())
 		errCh <- err
 		return
 	}
-	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.cfg.HFToken))
+	req.Header.Add("Authorization", "Bearer "+r.cfg.HFToken)
 	resp, err := http.DefaultClient.Do(req)
 	// nolint
 	// resp, err := httpClient.Post(cfg.EmbedURL, "application/json", bytes.NewReader(payload))
@@ -179,6 +203,7 @@ func (r *RAG) fecthEmbHF(lines []string, errCh chan error, vectorCh chan<- []mod
 			Embeddings: e,
 			RawText:    lines[i],
 			Slug:       slug,
+			FileName:   filename,
 		}
 		vectors[i] = vector
 	}
@@ -201,7 +226,7 @@ func (r *RAG) LineToVector(line string) ([]float32, error) {
 		r.logger.Error("failed to create new req", "err:", err.Error())
 		return nil, err
 	}
-	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.cfg.HFToken))
+	req.Header.Add("Authorization", "Bearer "+r.cfg.HFToken)
 	resp, err := http.DefaultClient.Do(req)
 	// resp, err := req.Post(r.cfg.EmbedURL, "application/json", bytes.NewReader(payload))
 	if err != nil {
@@ -228,15 +253,14 @@ func (r *RAG) LineToVector(line string) ([]float32, error) {
 	return emb[0], nil
 }
 
-// func (r *RAG) saveLine(topic, line string, emb *models.EmbeddingResp) error {
-// 	row := &models.VectorRow{
-// 		Embeddings: emb.Embedding,
-// 		Slug:       topic,
-// 		RawText:    line,
-// 	}
-// 	return r.store.WriteVector(row)
-// }
-
 func (r *RAG) SearchEmb(emb *models.EmbeddingResp) ([]models.VectorRow, error) {
 	return r.store.SearchClosest(emb.Embedding)
 }
+
+func (r *RAG) ListLoaded() ([]string, error) {
+	return r.store.ListFiles()
+}
+
+func (r *RAG) RemoveFile(filename string) error {
+	return r.store.RemoveEmbByFileName(filename)
+}

storage/migrations/002_add_vector.up.sql

@@ -1,11 +1,12 @@
-CREATE VIRTUAL TABLE IF NOT EXISTS embeddings USING vec0(
-    embedding FLOAT[5120],
-    slug TEXT NOT NULL,
-    raw_text TEXT PRIMARY KEY,
-);
+--CREATE VIRTUAL TABLE IF NOT EXISTS embeddings_5120 USING vec0(
+--    embedding FLOAT[5120],
+--    slug TEXT NOT NULL,
+--    raw_text TEXT PRIMARY KEY,
+--);
 
 CREATE VIRTUAL TABLE IF NOT EXISTS embeddings_384 USING vec0(
     embedding FLOAT[384],
     slug TEXT NOT NULL,
-    raw_text TEXT PRIMARY KEY
+    raw_text TEXT PRIMARY KEY,
+    filename TEXT NOT NULL DEFAULT ''
 );

storage/vector.go

@@ -12,17 +12,19 @@ import (
 type VectorRepo interface {
 	WriteVector(*models.VectorRow) error
 	SearchClosest(q []float32) ([]models.VectorRow, error)
+	ListFiles() ([]string, error)
+	RemoveEmbByFileName(filename string) error
 }
 
 var (
-	vecTableName    = "embeddings"
+	vecTableName5120 = "embeddings_5120"
 	vecTableName384  = "embeddings_384"
 )
 
 func fetchTableName(emb []float32) (string, error) {
 	switch len(emb) {
 	case 5120:
-		return vecTableName, nil
+		return vecTableName5120, nil
 	case 384:
 		return vecTableName384, nil
 	default:
@@ -36,7 +38,7 @@ func (p ProviderSQL) WriteVector(row *models.VectorRow) error {
 		return err
 	}
 	stmt, _, err := p.s3Conn.Prepare(
-		fmt.Sprintf("INSERT INTO %s(embedding, slug, raw_text) VALUES (?, ?, ?)", tableName))
+		fmt.Sprintf("INSERT INTO %s(embedding, slug, raw_text, filename) VALUES (?, ?, ?, ?)", tableName))
 	if err != nil {
 		p.logger.Error("failed to prep a stmt", "error", err)
 		return err
@@ -66,6 +68,10 @@ func (p ProviderSQL) WriteVector(row *models.VectorRow) error {
 		p.logger.Error("failed to bind", "error", err)
 		return err
 	}
+	if err := stmt.BindText(4, row.FileName); err != nil {
+		p.logger.Error("failed to bind", "error", err)
+		return err
+	}
 	err = stmt.Exec()
 	if err != nil {
 		return err
@@ -87,11 +93,12 @@ func (p ProviderSQL) SearchClosest(q []float32) ([]models.VectorRow, error) {
 			distance,
 			embedding,
 			slug,
-			raw_text
+			raw_text,
+			filename
 		FROM %s
 		WHERE embedding MATCH ?
 		ORDER BY distance
-		LIMIT 4
+		LIMIT 3
 	`, tableName))
 	if err != nil {
 		return nil, err
@@ -112,6 +119,7 @@ func (p ProviderSQL) SearchClosest(q []float32) ([]models.VectorRow, error) {
 		res.Embeddings = decodeUnsafe(emb)
 		res.Slug = stmt.ColumnText(2)
 		res.RawText = stmt.ColumnText(3)
+		res.FileName = stmt.ColumnText(4)
 		resp = append(resp, res)
 	}
 	if err := stmt.Err(); err != nil {
@@ -123,3 +131,33 @@ func (p ProviderSQL) SearchClosest(q []float32) ([]models.VectorRow, error) {
 	}
 	return resp, nil
 }
+
+func (p ProviderSQL) ListFiles() ([]string, error) {
+	q := fmt.Sprintf("SELECT filename FROM %s GROUP BY filename", vecTableName384)
+	stmt, _, err := p.s3Conn.Prepare(q)
+	if err != nil {
+		return nil, err
+	}
+	defer stmt.Close()
+	resp := []string{}
+	for stmt.Step() {
+		resp = append(resp, stmt.ColumnText(0))
+	}
+	if err := stmt.Err(); err != nil {
+		return nil, err
+	}
+	return resp, nil
+}
+
+func (p ProviderSQL) RemoveEmbByFileName(filename string) error {
+	q := fmt.Sprintf("DELETE FROM %s WHERE filename = ?", vecTableName384)
+	stmt, _, err := p.s3Conn.Prepare(q)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+	if err := stmt.BindText(1, filename); err != nil {
+		return err
+	}
+	return stmt.Exec()
+}

tables.go

@@ -0,0 +1,206 @@
+package main
+
+import (
+	"os"
+	"path"
+
+	"github.com/gdamore/tcell/v2"
+	"github.com/rivo/tview"
+)
+
+func makeChatTable(chatList []string) *tview.Table {
+	actions := []string{"load", "rename", "delete"}
+	rows, cols := len(chatList), len(actions)+1
+	chatActTable := tview.NewTable().
+		SetBorders(true)
+	for r := 0; r < rows; r++ {
+		for c := 0; c < cols; c++ {
+			color := tcell.ColorWhite
+			if c < 1 {
+				chatActTable.SetCell(r, c,
+					tview.NewTableCell(chatList[r]).
+						SetTextColor(color).
+						SetAlign(tview.AlignCenter))
+			} else {
+				chatActTable.SetCell(r, c,
+					tview.NewTableCell(actions[c-1]).
+						SetTextColor(color).
+						SetAlign(tview.AlignCenter))
+			}
+		}
+	}
+	chatActTable.Select(0, 0).SetFixed(1, 1).SetDoneFunc(func(key tcell.Key) {
+		if key == tcell.KeyEsc || key == tcell.KeyF1 {
+			pages.RemovePage(historyPage)
+			return
+		}
+		if key == tcell.KeyEnter {
+			chatActTable.SetSelectable(true, true)
+		}
+	}).SetSelectedFunc(func(row int, column int) {
+		tc := chatActTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		chatActTable.SetSelectable(false, false)
+		selectedChat := chatList[row]
+		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
+		switch tc.Text {
+		case "load":
+			history, err := loadHistoryChat(selectedChat)
+			if err != nil {
+				logger.Error("failed to read history file", "chat", selectedChat)
+				pages.RemovePage(historyPage)
+				return
+			}
+			chatBody.Messages = history
+			textView.SetText(chatToText(cfg.ShowSys))
+			activeChatName = selectedChat
+			pages.RemovePage(historyPage)
+			colorText()
+			updateStatusLine()
+			return
+		case "rename":
+			pages.RemovePage(historyPage)
+			pages.AddPage(renamePage, renameWindow, true, true)
+			return
+		case "delete":
+			sc, ok := chatMap[selectedChat]
+			if !ok {
+				// no chat found
+				pages.RemovePage(historyPage)
+				return
+			}
+			if err := store.RemoveChat(sc.ID); err != nil {
+				logger.Error("failed to remove chat from db", "chat_id", sc.ID, "chat_name", sc.Name)
+			}
+			if err := notifyUser("chat deleted", selectedChat+" was deleted"); err != nil {
+				logger.Error("failed to send notification", "error", err)
+			}
+			pages.RemovePage(historyPage)
+			return
+		default:
+			pages.RemovePage(historyPage)
+			return
+		}
+	})
+	return chatActTable
+}
+
+func makeRAGTable(fileList []string) *tview.Table {
+	actions := []string{"load", "delete"}
+	rows, cols := len(fileList), len(actions)+1
+	fileTable := tview.NewTable().
+		SetBorders(true)
+	for r := 0; r < rows; r++ {
+		for c := 0; c < cols; c++ {
+			color := tcell.ColorWhite
+			if c < 1 {
+				fileTable.SetCell(r, c,
+					tview.NewTableCell(fileList[r]).
+						SetTextColor(color).
+						SetAlign(tview.AlignCenter))
+			} else {
+				fileTable.SetCell(r, c,
+					tview.NewTableCell(actions[c-1]).
+						SetTextColor(color).
+						SetAlign(tview.AlignCenter))
+			}
+		}
+	}
+	fileTable.Select(0, 0).SetFixed(1, 1).SetDoneFunc(func(key tcell.Key) {
+		if key == tcell.KeyEsc || key == tcell.KeyF1 {
+			pages.RemovePage(RAGPage)
+			return
+		}
+		if key == tcell.KeyEnter {
+			fileTable.SetSelectable(true, true)
+		}
+	}).SetSelectedFunc(func(row int, column int) {
+		defer pages.RemovePage(RAGPage)
+		tc := fileTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		fileTable.SetSelectable(false, false)
+		fpath := fileList[row]
+		// notification := fmt.Sprintf("chat: %s; action: %s", fpath, tc.Text)
+		switch tc.Text {
+		case "load":
+			fpath = path.Join(cfg.RAGDir, fpath)
+			if err := ragger.LoadRAG(fpath); err != nil {
+				logger.Error("failed to embed file", "chat", fpath, "error", err)
+				// pages.RemovePage(RAGPage)
+				return
+			}
+			pages.RemovePage(RAGPage)
+			colorText()
+			updateStatusLine()
+			return
+		case "delete":
+			fpath = path.Join(cfg.RAGDir, fpath)
+			if err := os.Remove(fpath); err != nil {
+				logger.Error("failed to delete file", "filename", fpath, "error", err)
+				return
+			}
+			if err := notifyUser("chat deleted", fpath+" was deleted"); err != nil {
+				logger.Error("failed to send notification", "error", err)
+			}
+			return
+		default:
+			// pages.RemovePage(RAGPage)
+			return
+		}
+	})
+	return fileTable
+}
+
+func makeLoadedRAGTable(fileList []string) *tview.Table {
+	actions := []string{"delete"}
+	rows, cols := len(fileList), len(actions)+1
+	fileTable := tview.NewTable().
+		SetBorders(true)
+	for r := 0; r < rows; r++ {
+		for c := 0; c < cols; c++ {
+			color := tcell.ColorWhite
+			if c < 1 {
+				fileTable.SetCell(r, c,
+					tview.NewTableCell(fileList[r]).
+						SetTextColor(color).
+						SetAlign(tview.AlignCenter))
+			} else {
+				fileTable.SetCell(r, c,
+					tview.NewTableCell(actions[c-1]).
+						SetTextColor(color).
+						SetAlign(tview.AlignCenter))
+			}
+		}
+	}
+	fileTable.Select(0, 0).SetFixed(1, 1).SetDoneFunc(func(key tcell.Key) {
+		if key == tcell.KeyEsc || key == tcell.KeyF1 {
+			pages.RemovePage(RAGPage)
+			return
+		}
+		if key == tcell.KeyEnter {
+			fileTable.SetSelectable(true, true)
+		}
+	}).SetSelectedFunc(func(row int, column int) {
+		defer pages.RemovePage(RAGPage)
+		tc := fileTable.GetCell(row, column)
+		tc.SetTextColor(tcell.ColorRed)
+		fileTable.SetSelectable(false, false)
+		fpath := fileList[row]
+		// notification := fmt.Sprintf("chat: %s; action: %s", fpath, tc.Text)
+		switch tc.Text {
+		case "delete":
+			if err := ragger.RemoveFile(fpath); err != nil {
+				logger.Error("failed to delete file", "filename", fpath, "error", err)
+				return
+			}
+			if err := notifyUser("chat deleted", fpath+" was deleted"); err != nil {
+				logger.Error("failed to send notification", "error", err)
+			}
+			return
+		default:
+			// pages.RemovePage(RAGPage)
+			return
+		}
+	})
+	return fileTable
+}

tui.go

@@ -5,7 +5,6 @@ import (
 	"elefant/pngmeta"
 	"fmt"
 	"os"
-	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -47,6 +46,9 @@ var (
 [yellow]F6[white]: interrupt bot resp
 [yellow]F7[white]: copy last msg to clipboard (linux xclip)
 [yellow]F8[white]: copy n msg to clipboard (linux xclip)
+[yellow]F10[white]: manage loaded rag files
+[yellow]F11[white]: switch RAGEnabled boolean
+[yellow]F12[white]: show this help page
 [yellow]Ctrl+s[white]: load new char/agent
 [yellow]Ctrl+e[white]: export chat to json file
 [yellow]Ctrl+n[white]: start a new chat
@@ -56,157 +58,6 @@ Press Enter to go back
 `
 )
 
-func makeChatTable(chatList []string) *tview.Table {
-	actions := []string{"load", "rename", "delete"}
-	rows, cols := len(chatList), len(actions)+1
-	chatActTable := tview.NewTable().
-		SetBorders(true)
-	for r := 0; r < rows; r++ {
-		for c := 0; c < cols; c++ {
-			color := tcell.ColorWhite
-			if c < 1 {
-				chatActTable.SetCell(r, c,
-					tview.NewTableCell(chatList[r]).
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter))
-			} else {
-				chatActTable.SetCell(r, c,
-					tview.NewTableCell(actions[c-1]).
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter))
-			}
-		}
-	}
-	chatActTable.Select(0, 0).SetFixed(1, 1).SetDoneFunc(func(key tcell.Key) {
-		if key == tcell.KeyEsc || key == tcell.KeyF1 {
-			pages.RemovePage(historyPage)
-			return
-		}
-		if key == tcell.KeyEnter {
-			chatActTable.SetSelectable(true, true)
-		}
-	}).SetSelectedFunc(func(row int, column int) {
-		tc := chatActTable.GetCell(row, column)
-		tc.SetTextColor(tcell.ColorRed)
-		chatActTable.SetSelectable(false, false)
-		selectedChat := chatList[row]
-		// notification := fmt.Sprintf("chat: %s; action: %s", selectedChat, tc.Text)
-		switch tc.Text {
-		case "load":
-			history, err := loadHistoryChat(selectedChat)
-			if err != nil {
-				logger.Error("failed to read history file", "chat", selectedChat)
-				pages.RemovePage(historyPage)
-				return
-			}
-			chatBody.Messages = history
-			textView.SetText(chatToText(cfg.ShowSys))
-			activeChatName = selectedChat
-			pages.RemovePage(historyPage)
-			colorText()
-			updateStatusLine()
-			return
-		case "rename":
-			pages.RemovePage(historyPage)
-			pages.AddPage(renamePage, renameWindow, true, true)
-			return
-		case "delete":
-			sc, ok := chatMap[selectedChat]
-			if !ok {
-				// no chat found
-				pages.RemovePage(historyPage)
-				return
-			}
-			if err := store.RemoveChat(sc.ID); err != nil {
-				logger.Error("failed to remove chat from db", "chat_id", sc.ID, "chat_name", sc.Name)
-			}
-			if err := notifyUser("chat deleted", selectedChat+" was deleted"); err != nil {
-				logger.Error("failed to send notification", "error", err)
-			}
-			pages.RemovePage(historyPage)
-			return
-		default:
-			pages.RemovePage(historyPage)
-			return
-		}
-	})
-	return chatActTable
-}
-
-func makeRAGTable(fileList []string) *tview.Table {
-	actions := []string{"load", "rename", "delete"}
-	rows, cols := len(fileList), len(actions)+1
-	chatActTable := tview.NewTable().
-		SetBorders(true)
-	for r := 0; r < rows; r++ {
-		for c := 0; c < cols; c++ {
-			color := tcell.ColorWhite
-			if c < 1 {
-				chatActTable.SetCell(r, c,
-					tview.NewTableCell(fileList[r]).
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter))
-			} else {
-				chatActTable.SetCell(r, c,
-					tview.NewTableCell(actions[c-1]).
-						SetTextColor(color).
-						SetAlign(tview.AlignCenter))
-			}
-		}
-	}
-	chatActTable.Select(0, 0).SetFixed(1, 1).SetDoneFunc(func(key tcell.Key) {
-		if key == tcell.KeyEsc || key == tcell.KeyF1 {
-			pages.RemovePage(RAGPage)
-			return
-		}
-		if key == tcell.KeyEnter {
-			chatActTable.SetSelectable(true, true)
-		}
-	}).SetSelectedFunc(func(row int, column int) {
-		tc := chatActTable.GetCell(row, column)
-		tc.SetTextColor(tcell.ColorRed)
-		chatActTable.SetSelectable(false, false)
-		fpath := fileList[row]
-		// notification := fmt.Sprintf("chat: %s; action: %s", fpath, tc.Text)
-		switch tc.Text {
-		case "load":
-			fpath = path.Join(cfg.RAGDir, fpath)
-			if err := ragger.LoadRAG(fpath); err != nil {
-				logger.Error("failed to embed file", "chat", fpath, "error", err)
-				pages.RemovePage(RAGPage)
-				return
-			}
-			pages.RemovePage(RAGPage)
-			colorText()
-			updateStatusLine()
-			return
-		case "rename":
-			pages.RemovePage(RAGPage)
-			pages.AddPage(renamePage, renameWindow, true, true)
-			return
-		case "delete":
-			sc, ok := chatMap[fpath]
-			if !ok {
-				// no chat found
-				pages.RemovePage(RAGPage)
-				return
-			}
-			if err := store.RemoveChat(sc.ID); err != nil {
-				logger.Error("failed to remove chat from db", "chat_id", sc.ID, "chat_name", sc.Name)
-			}
-			if err := notifyUser("chat deleted", fpath+" was deleted"); err != nil {
-				logger.Error("failed to send notification", "error", err)
-			}
-			pages.RemovePage(RAGPage)
-			return
-		default:
-			pages.RemovePage(RAGPage)
-			return
-		}
-	})
-	return chatActTable
-}
-
 // // code block colors get interrupted by " & *
 // func codeBlockColor(text string) string {
 // 	fi := strings.Index(text, "```")
@@ -360,8 +211,11 @@ func init() {
 		SetAcceptanceFunc(tview.InputFieldInteger).
 		SetDoneFunc(func(key tcell.Key) {
 			pages.RemovePage(indexPage)
+			colorText()
+			updateStatusLine()
 		})
 	indexPickWindow.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		defer indexPickWindow.SetText("")
 		switch event.Key() {
 		case tcell.KeyBackspace:
 			return event
@@ -534,9 +388,26 @@ func init() {
 			pages.AddPage(indexPage, indexPickWindow, true, true)
 			return nil
 		}
+		if event.Key() == tcell.KeyF10 {
+			// list rag loaded in db
+			loadedFiles, err := ragger.ListLoaded()
+			if err != nil {
+				logger.Error("failed to list regfiles in db", "error", err)
+				return nil
+			}
+			if len(loadedFiles) == 0 {
+				if err := notifyUser("loaded RAG", "no files in db"); err != nil {
+					logger.Error("failed to send notification", "error", err)
+				}
+				return nil
+			}
+			dbRAGTable := makeLoadedRAGTable(loadedFiles)
+			pages.AddPage(RAGPage, dbRAGTable, true, true)
+			return nil
+		}
 		if event.Key() == tcell.KeyF11 {
 			// xor
-			cfg.RAGEnabled = cfg.RAGEnabled != true
+			cfg.RAGEnabled = !cfg.RAGEnabled
 			updateStatusLine()
 			return nil
 		}
@@ -604,8 +475,6 @@ func init() {
 			position.SetText(fmt.Sprintf(indexLine, botRespMode, cfg.AssistantRole, activeChatName))
 			// read all text into buffer
 			msgText := textArea.GetText()
-			// TODO: check whose message was latest (user icon / assistant)
-			// in order to decide if assistant new icon is needed
 			nl := "\n"
 			prevText := textView.GetText(true)
 			// strings.LastIndex()