grail/gf-lt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: open router impl

Browse Source
This commit is contained in:
Grail Finder
2025-08-07 12:18:01 +03:00
parent 9b2558ffe8
commit 813cb49d36
8 changed files with 385 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.golangci.yml
View File

@@ -10,7 +10,6 @@ linters:
- fatcontext
- govet
- ineffassign
- noctx
- perfsprint
- prealloc
- staticcheck

41
bot.go
View File

@@ -52,6 +52,14 @@ var (
"min_p": 0.05,
"n_predict": -1.0,
}
ORFreeModels = []string{
"google/gemini-2.0-flash-exp:free",
"deepseek/deepseek-chat-v3-0324:free",
"mistralai/mistral-small-3.2-24b-instruct:free",
"qwen/qwen3-14b:free",
"google/gemma-3-27b-it:free",
"meta-llama/llama-3.3-70b-instruct:free",
}
)
func createClient(connectTimeout time.Duration) *http.Client {
@@ -124,6 +132,24 @@ func fetchDSBalance() *models.DSBalance {
return &resp
}
func fetchORModels(free bool) ([]string, error) {
resp, err := http.Get("https://openrouter.ai/api/v1/models")
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
err := fmt.Errorf("failed to fetch or models; status: %s", resp.Status)
return nil, err
}
data := &models.ORModels{}
if err := json.NewDecoder(resp.Body).Decode(data); err != nil {
return nil, err
}
freeModels := data.ListModels(free)
return freeModels, nil
}
func sendMsgToLLM(body io.Reader) {
choseChunkParser()
bodyBytes, _ := io.ReadAll(body)
@@ -143,7 +169,7 @@ func sendMsgToLLM(body io.Reader) {
}
req.Header.Add("Accept", "application/json")
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", "Bearer "+cfg.DeepSeekToken)
req.Header.Add("Authorization", "Bearer "+chunkParser.GetToken())
req.Header.Set("Content-Length", strconv.Itoa(len(bodyBytes)))
req.Header.Set("Accept-Encoding", "gzip")
// nolint
@@ -196,6 +222,9 @@ func sendMsgToLLM(body io.Reader) {
streamDone <- true
break
}
if bytes.Equal(line, []byte("ROUTER PROCESSING\n")) {
continue
}
content, stop, err = chunkParser.ParseChunk(line)
if err != nil {
logger.Error("error parsing response body", "error", err,
@@ -408,7 +437,7 @@ func chatToTextSlice(showSys bool) []string {
if !showSys && (msg.Role != cfg.AssistantRole && msg.Role != cfg.UserRole) {
continue
}
resp[i] = msg.ToText(i, cfg)
resp[i] = msg.ToText(i)
}
return resp
}
@@ -524,6 +553,14 @@ func init() {
playerOrder = []string{cfg.UserRole, cfg.AssistantRole, cfg.CluedoRole2}
cluedoState = extra.CluedoPrepCards(playerOrder)
}
if cfg.OpenRouterToken != "" {
ORModels, err := fetchORModels(true)
if err != nil {
logger.Error("failed to fetch or models", "error", err)
} else {
ORFreeModels = ORModels
}
}
choseChunkParser()
httpClient = createClient(time.Second * 15)
if cfg.TTS_ENABLED {

2
config.example.toml
View File

@@ -1,5 +1,7 @@
ChatAPI = "http://localhost:8080/v1/chat/completions"
CompletionAPI = "http://localhost:8080/completion"
OpenRouterCompletionAPI = "https://openrouter.ai/api/v1/completions"
OpenRouterChatAPI = "https://openrouter.ai/api/v1/chat/completions"
EmbedURL = "http://localhost:8080/v1/embeddings"
ShowSys = true
LogFile = "log.txt"

17
config/config.go
View File

@@ -41,6 +41,11 @@ type Config struct {
DeepSeekToken string `toml:"DeepSeekToken"`
DeepSeekModel string `toml:"DeepSeekModel"`
ApiLinks []string
// openrouter
OpenRouterChatAPI string `toml:"OpenRouterChatAPI"`
OpenRouterCompletionAPI string `toml:"OpenRouterCompletionAPI"`
OpenRouterToken string `toml:"OpenRouterToken"`
OpenRouterModel string `toml:"OpenRouterModel"`
// TTS
TTS_URL string `toml:"TTS_URL"`
TTS_ENABLED bool `toml:"TTS_ENABLED"`
@@ -62,6 +67,8 @@ func LoadConfigOrDefault(fn string) *Config {
config.CompletionAPI = "http://localhost:8080/completion"
config.DeepSeekCompletionAPI = "https://api.deepseek.com/beta/completions"
config.DeepSeekChatAPI = "https://api.deepseek.com/chat/completions"
config.OpenRouterCompletionAPI = "https://openrouter.ai/api/v1/completions"
config.OpenRouterChatAPI = "https://openrouter.ai/api/v1/chat/completions"
config.RAGEnabled = false
config.EmbedURL = "http://localhost:8080/v1/embiddings"
config.ShowSys = true
@@ -81,10 +88,12 @@ func LoadConfigOrDefault(fn string) *Config {
}
config.CurrentAPI = config.ChatAPI
config.APIMap = map[string]string{
config.ChatAPI: config.CompletionAPI,
config.CompletionAPI: config.DeepSeekChatAPI,
config.DeepSeekChatAPI: config.DeepSeekCompletionAPI,
config.DeepSeekCompletionAPI: config.ChatAPI,
config.ChatAPI: config.CompletionAPI,
config.CompletionAPI: config.DeepSeekChatAPI,
config.DeepSeekChatAPI: config.DeepSeekCompletionAPI,
config.DeepSeekCompletionAPI: config.OpenRouterCompletionAPI,
config.OpenRouterCompletionAPI: config.OpenRouterChatAPI,
config.OpenRouterChatAPI: config.ChatAPI,
}
for _, el := range []string{config.ChatAPI, config.CompletionAPI, config.DeepSeekChatAPI, config.DeepSeekCompletionAPI} {
if el != "" {

178
llm.go
View File

@@ -11,6 +11,7 @@ import (
type ChunkParser interface {
ParseChunk([]byte) (string, bool, error)
FormMsg(msg, role string, cont bool) (io.Reader, error)
GetToken() string
}
func choseChunkParser() {
@@ -32,15 +33,17 @@ func choseChunkParser() {
chunkParser = DeepSeekerChat{}
logger.Debug("chosen deepseekerchat", "link", cfg.CurrentAPI)
return
case "https://openrouter.ai/api/v1/completions":
chunkParser = OpenRouterCompletion{}
logger.Debug("chosen openroutercompletion", "link", cfg.CurrentAPI)
return
case "https://openrouter.ai/api/v1/chat/completions":
chunkParser = OpenRouterChat{}
logger.Debug("chosen openrouterchat", "link", cfg.CurrentAPI)
return
default:
chunkParser = LlamaCPPeer{}
}
// if strings.Contains(cfg.CurrentAPI, "chat") {
// logger.Debug("chosen chat parser")
// chunkParser = OpenAIer{}
// return
// }
// logger.Debug("chosen llamacpp /completion parser")
}
type LlamaCPPeer struct {
@@ -51,6 +54,16 @@ type DeepSeekerCompletion struct {
}
type DeepSeekerChat struct {
}
type OpenRouterCompletion struct {
Model string
}
type OpenRouterChat struct {
Model string
}
func (lcp LlamaCPPeer) GetToken() string {
return ""
}
func (lcp LlamaCPPeer) FormMsg(msg, role string, resume bool) (io.Reader, error) {
logger.Debug("formmsg llamacppeer", "link", cfg.CurrentAPI)
@@ -88,10 +101,10 @@ func (lcp LlamaCPPeer) FormMsg(msg, role string, resume bool) (io.Reader, error)
logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
"msg", msg, "resume", resume, "prompt", prompt)
var payload any
payload = models.NewLCPReq(prompt, cfg, defaultLCPProps, chatBody.MakeStopSlice())
if strings.Contains(chatBody.Model, "deepseek") {
payload = models.NewLCPReq(prompt, defaultLCPProps, chatBody.MakeStopSlice())
if strings.Contains(chatBody.Model, "deepseek") { // TODO: why?
payload = models.NewDSCompletionReq(prompt, chatBody.Model,
defaultLCPProps["temp"], cfg, chatBody.MakeStopSlice())
defaultLCPProps["temp"], chatBody.MakeStopSlice())
}
data, err := json.Marshal(payload)
if err != nil {
@@ -116,6 +129,10 @@ func (lcp LlamaCPPeer) ParseChunk(data []byte) (string, bool, error) {
return llmchunk.Content, false, nil
}
func (op OpenAIer) GetToken() string {
return ""
}
func (op OpenAIer) ParseChunk(data []byte) (string, bool, error) {
llmchunk := models.LLMRespChunk{}
if err := json.Unmarshal(data, &llmchunk); err != nil {
@@ -177,6 +194,10 @@ func (ds DeepSeekerCompletion) ParseChunk(data []byte) (string, bool, error) {
return llmchunk.Choices[0].Text, false, nil
}
func (ds DeepSeekerCompletion) GetToken() string {
return cfg.DeepSeekToken
}
func (ds DeepSeekerCompletion) FormMsg(msg, role string, resume bool) (io.Reader, error) {
logger.Debug("formmsg deepseekercompletion", "link", cfg.CurrentAPI)
if msg != "" { // otherwise let the bot to continue
@@ -213,7 +234,7 @@ func (ds DeepSeekerCompletion) FormMsg(msg, role string, resume bool) (io.Reader
logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
"msg", msg, "resume", resume, "prompt", prompt)
payload := models.NewDSCompletionReq(prompt, chatBody.Model,
defaultLCPProps["temp"], cfg, chatBody.MakeStopSlice())
defaultLCPProps["temp"], chatBody.MakeStopSlice())
data, err := json.Marshal(payload)
if err != nil {
logger.Error("failed to form a msg", "error", err)
@@ -240,6 +261,10 @@ func (ds DeepSeekerChat) ParseChunk(data []byte) (string, bool, error) {
return llmchunk.Choices[0].Delta.Content, false, nil
}
func (ds DeepSeekerChat) GetToken() string {
return cfg.DeepSeekToken
}
func (ds DeepSeekerChat) FormMsg(msg, role string, resume bool) (io.Reader, error) {
logger.Debug("formmsg deepseekerchat", "link", cfg.CurrentAPI)
if cfg.ToolUse && !resume {
@@ -286,3 +311,136 @@ func (ds DeepSeekerChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
}
return bytes.NewReader(data), nil
}
// openrouter
func (or OpenRouterCompletion) ParseChunk(data []byte) (string, bool, error) {
llmchunk := models.OpenRouterCompletionResp{}
if err := json.Unmarshal(data, &llmchunk); err != nil {
logger.Error("failed to decode", "error", err, "line", string(data))
return "", false, err
}
content := llmchunk.Choices[len(llmchunk.Choices)-1].Text
if llmchunk.Choices[len(llmchunk.Choices)-1].FinishReason == "stop" {
if content != "" {
logger.Error("text inside of finish llmchunk", "chunk", llmchunk)
}
return content, true, nil
}
return content, false, nil
}
func (or OpenRouterCompletion) GetToken() string {
return cfg.OpenRouterToken
}
func (or OpenRouterCompletion) FormMsg(msg, role string, resume bool) (io.Reader, error) {
logger.Debug("formmsg openroutercompletion", "link", cfg.CurrentAPI)
if msg != "" { // otherwise let the bot to continue
newMsg := models.RoleMsg{Role: role, Content: msg}
chatBody.Messages = append(chatBody.Messages, newMsg)
// if rag
if cfg.RAGEnabled {
ragResp, err := chatRagUse(newMsg.Content)
if err != nil {
logger.Error("failed to form a rag msg", "error", err)
return nil, err
}
ragMsg := models.RoleMsg{Role: cfg.ToolRole, Content: ragResp}
chatBody.Messages = append(chatBody.Messages, ragMsg)
}
}
if cfg.ToolUse && !resume {
// add to chat body
chatBody.Messages = append(chatBody.Messages, models.RoleMsg{Role: cfg.ToolRole, Content: toolSysMsg})
}
messages := make([]string, len(chatBody.Messages))
for i, m := range chatBody.Messages {
messages[i] = m.ToPrompt()
}
prompt := strings.Join(messages, "\n")
// strings builder?
if !resume {
botMsgStart := "\n" + cfg.AssistantRole + ":\n"
prompt += botMsgStart
}
if cfg.ThinkUse && !cfg.ToolUse {
prompt += "<think>"
}
logger.Debug("checking prompt for /completion", "tool_use", cfg.ToolUse,
"msg", msg, "resume", resume, "prompt", prompt)
payload := models.NewOpenRouterCompletionReq(chatBody.Model, prompt, defaultLCPProps, chatBody.MakeStopSlice())
data, err := json.Marshal(payload)
if err != nil {
logger.Error("failed to form a msg", "error", err)
return nil, err
}
return bytes.NewReader(data), nil
}
// chat
func (or OpenRouterChat) ParseChunk(data []byte) (string, bool, error) {
llmchunk := models.OpenRouterChatResp{}
if err := json.Unmarshal(data, &llmchunk); err != nil {
logger.Error("failed to decode", "error", err, "line", string(data))
return "", false, err
}
content := llmchunk.Choices[len(llmchunk.Choices)-1].Delta.Content
if llmchunk.Choices[len(llmchunk.Choices)-1].FinishReason == "stop" {
if content != "" {
logger.Error("text inside of finish llmchunk", "chunk", llmchunk)
}
return content, true, nil
}
return content, false, nil
}
func (or OpenRouterChat) GetToken() string {
return cfg.OpenRouterToken
}
func (or OpenRouterChat) FormMsg(msg, role string, resume bool) (io.Reader, error) {
logger.Debug("formmsg open router completion", "link", cfg.CurrentAPI)
if cfg.ToolUse && !resume {
// prompt += "\n" + cfg.ToolRole + ":\n" + toolSysMsg
// add to chat body
chatBody.Messages = append(chatBody.Messages, models.RoleMsg{Role: cfg.ToolRole, Content: toolSysMsg})
}
if msg != "" { // otherwise let the bot continue
newMsg := models.RoleMsg{Role: role, Content: msg}
chatBody.Messages = append(chatBody.Messages, newMsg)
// if rag
if cfg.RAGEnabled {
ragResp, err := chatRagUse(newMsg.Content)
if err != nil {
logger.Error("failed to form a rag msg", "error", err)
return nil, err
}
ragMsg := models.RoleMsg{Role: cfg.ToolRole, Content: ragResp}
chatBody.Messages = append(chatBody.Messages, ragMsg)
}
}
// Create copy of chat body with standardized user role
// modifiedBody := *chatBody
bodyCopy := &models.ChatBody{
Messages: make([]models.RoleMsg, len(chatBody.Messages)),
Model: chatBody.Model,
Stream: chatBody.Stream,
}
// modifiedBody.Messages = make([]models.RoleMsg, len(chatBody.Messages))
for i, msg := range chatBody.Messages {
logger.Debug("checking roles", "#", i, "role", msg.Role)
if msg.Role == cfg.UserRole || i == 1 {
bodyCopy.Messages[i].Role = "user"
logger.Debug("replaced role in body", "#", i)
} else {
bodyCopy.Messages[i] = msg
}
}
dsBody := models.NewDSCharReq(*bodyCopy)
data, err := json.Marshal(dsBody)
if err != nil {
logger.Error("failed to form a msg", "error", err)
return nil, err
}
return bytes.NewReader(data), nil
}

160
models/models.go
View File

@@ -2,7 +2,6 @@ package models
import (
"fmt"
"gf-lt/config"
"strings"
)
@@ -56,7 +55,7 @@ type RoleMsg struct {
Content string `json:"content"`
}
func (m RoleMsg) ToText(i int, cfg *config.Config) string {
func (m RoleMsg) ToText(i int) string {
icon := fmt.Sprintf("(%d)", i)
// check if already has role annotation (/completion makes them)
if !strings.HasPrefix(m.Content, m.Role+":") {
@@ -185,7 +184,7 @@ type DSCompletionReq struct {
TopP float32 `json:"top_p"`
}
func NewDSCompletionReq(prompt, model string, temp float32, cfg *config.Config, stopSlice []string) DSCompletionReq {
func NewDSCompletionReq(prompt, model string, temp float32, stopSlice []string) DSCompletionReq {
return DSCompletionReq{
Model: model,
Prompt: prompt,
@@ -334,7 +333,7 @@ type LlamaCPPReq struct {
// Samplers string `json:"samplers"`
}
func NewLCPReq(prompt string, cfg *config.Config, props map[string]float32, stopStrings []string) LlamaCPPReq {
func NewLCPReq(prompt string, props map[string]float32, stopStrings []string) LlamaCPPReq {
return LlamaCPPReq{
Stream: true,
Prompt: prompt,
@@ -362,3 +361,156 @@ type DSBalance struct {
ToppedUpBalance string `json:"topped_up_balance"`
} `json:"balance_infos"`
}
// openrouter
// https://openrouter.ai/docs/api-reference/completion
type OpenRouterCompletionReq struct {
Model string `json:"model"`
Prompt string `json:"prompt"`
Stream bool `json:"stream"`
Temperature float32 `json:"temperature"`
Stop []string `json:"stop"` // not present in docs
MinP float32 `json:"min_p"`
NPredict int32 `json:"max_tokens"`
}
func NewOpenRouterCompletionReq(model, prompt string, props map[string]float32, stopStrings []string) OpenRouterCompletionReq {
return OpenRouterCompletionReq{
Stream: true,
Prompt: prompt,
Temperature: props["temperature"],
MinP: props["min_p"],
NPredict: int32(props["n_predict"]),
Stop: stopStrings,
Model: model,
}
}
type OpenRouterChatReq struct {
Messages []RoleMsg `json:"messages"`
Model string `json:"model"`
Stream bool `json:"stream"`
Temperature float32 `json:"temperature"`
MinP float32 `json:"min_p"`
NPredict int32 `json:"max_tokens"`
}
func NewOpenRouterChatReq(cb ChatBody, props map[string]float32) OpenRouterChatReq {
return OpenRouterChatReq{
Messages: cb.Messages,
Model: cb.Model,
Stream: cb.Stream,
Temperature: props["temperature"],
MinP: props["min_p"],
NPredict: int32(props["n_predict"]),
}
}
type OpenRouterChatRespNonStream struct {
ID string `json:"id"`
Provider string `json:"provider"`
Model string `json:"model"`
Object string `json:"object"`
Created int `json:"created"`
Choices []struct {
Logprobs any `json:"logprobs"`
FinishReason string `json:"finish_reason"`
NativeFinishReason string `json:"native_finish_reason"`
Index int `json:"index"`
Message struct {
Role string `json:"role"`
Content string `json:"content"`
Refusal any `json:"refusal"`
Reasoning any `json:"reasoning"`
} `json:"message"`
} `json:"choices"`
Usage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
} `json:"usage"`
}
type OpenRouterChatResp struct {
ID string `json:"id"`
Provider string `json:"provider"`
Model string `json:"model"`
Object string `json:"object"`
Created int `json:"created"`
Choices []struct {
Index int `json:"index"`
Delta struct {
Role string `json:"role"`
Content string `json:"content"`
} `json:"delta"`
FinishReason string `json:"finish_reason"`
NativeFinishReason string `json:"native_finish_reason"`
Logprobs any `json:"logprobs"`
} `json:"choices"`
}
type OpenRouterCompletionResp struct {
ID string `json:"id"`
Provider string `json:"provider"`
Model string `json:"model"`
Object string `json:"object"`
Created int `json:"created"`
Choices []struct {
Text string `json:"text"`
FinishReason string `json:"finish_reason"`
NativeFinishReason string `json:"native_finish_reason"`
Logprobs any `json:"logprobs"`
} `json:"choices"`
}
type ORModel struct {
ID string `json:"id"`
CanonicalSlug string `json:"canonical_slug"`
HuggingFaceID string `json:"hugging_face_id"`
Name string `json:"name"`
Created int `json:"created"`
Description string `json:"description"`
ContextLength int `json:"context_length"`
Architecture struct {
Modality string `json:"modality"`
InputModalities []string `json:"input_modalities"`
OutputModalities []string `json:"output_modalities"`
Tokenizer string `json:"tokenizer"`
InstructType any `json:"instruct_type"`
} `json:"architecture"`
Pricing struct {
Prompt string `json:"prompt"`
Completion string `json:"completion"`
Request string `json:"request"`
Image string `json:"image"`
Audio string `json:"audio"`
WebSearch string `json:"web_search"`
InternalReasoning string `json:"internal_reasoning"`
} `json:"pricing,omitempty"`
TopProvider struct {
ContextLength int `json:"context_length"`
MaxCompletionTokens int `json:"max_completion_tokens"`
IsModerated bool `json:"is_moderated"`
} `json:"top_provider"`
PerRequestLimits any `json:"per_request_limits"`
SupportedParameters []string `json:"supported_parameters"`
}
type ORModels struct {
Data []ORModel `json:"data"`
}
func (orm *ORModels) ListModels(free bool) []string {
resp := []string{}
for _, model := range orm.Data {
if free {
if model.Pricing.Prompt == "0" && model.Pricing.Request == "0" &&
model.Pricing.Completion == "0" {
resp = append(resp, model.ID)
}
} else {
resp = append(resp, model.ID)
}
}
return resp
}

1
tables.go
View File

@@ -144,7 +144,6 @@ func makeChatTable(chatMap map[string]models.Chat) *tview.Table {
return
}
// Reload card from disk
newCard := &models.CharCard{}
newCard, err := pngmeta.ReadCard(cc.FilePath, cfg.UserRole)
if err != nil {
logger.Error("failed to reload charcard", "path", cc.FilePath, "error", err)

13
tui.go
View File

@@ -241,6 +241,8 @@ func setLogLevel(sl string) {
func makePropsForm(props map[string]float32) *tview.Form {
// https://github.com/rivo/tview/commit/0a18dea458148770d212d348f656988df75ff341
// no way to close a form by a key press; a shame.
modelList := []string{chatBody.Model, "deepseek-chat", "deepseek-reasoner"}
modelList = append(modelList, ORFreeModels...)
form := tview.NewForm().
AddTextView("Notes", "Props for llamacpp completion call", 40, 2, true, false).
AddCheckbox("Insert <think> (/completion only)", cfg.ThinkUse, func(checked bool) {
@@ -253,7 +255,7 @@ func makePropsForm(props map[string]float32) *tview.Form {
}).AddDropDown("Select an api: ", slices.Insert(cfg.ApiLinks, 0, cfg.CurrentAPI), 0,
func(option string, optionIndex int) {
cfg.CurrentAPI = option
}).AddDropDown("Select a model: ", []string{chatBody.Model, "deepseek-chat", "deepseek-reasoner"}, 0,
}).AddDropDown("Select a model: ", modelList, 0,
func(option string, optionIndex int) {
chatBody.Model = option
}).AddDropDown("Write next message as: ", chatBody.ListRoles(), 0,
@@ -694,11 +696,10 @@ func init() {
return nil
}
cfg.CurrentAPI = newAPI
if strings.Contains(cfg.CurrentAPI, "deepseek") {
chatBody.Model = "deepseek-chat"
} else {
chatBody.Model = "local"
}
// // TODO: implement model pick
// if strings.Contains(cfg.CurrentAPI, "deepseek") {
// chatBody.Model = "deepseek-chat"
// }
choseChunkParser()
updateStatusLine()
return nil