grail/gf-lt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enha: or img attachment

Browse Source
This commit is contained in:
Grail Finder
2025-11-24 13:06:42 +03:00
parent 8a62e98789
commit b5ade05489
2 changed files with 202 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
llm.go
View File

@@ -3,8 +3,10 @@ package main
import (
"bytes"
"encoding/json"
"fmt"
"gf-lt/models"
"io"
"os"
"strings"
)
@@ -76,6 +78,13 @@ type OpenRouterChat struct {
Model string
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
func (lcp LlamaCPPeer) GetToken() string {
return ""
}
@@ -180,24 +189,29 @@ func (op OpenAIer) ParseChunk(data []byte) (*models.TextChunk, error) {
func (op OpenAIer) FormMsg(msg, role string, resume bool) (io.Reader, error) {
logger.Debug("formmsg openaier", "link", cfg.CurrentAPI)
// Capture the image attachment path at the beginning to avoid race conditions
// with API rotation that might clear the global variable
localImageAttachmentPath := imageAttachmentPath
if msg != "" { // otherwise let the bot continue
// Create the message with support for multimodal content
var newMsg models.RoleMsg
// Check if we have an image to add to this message
if imageAttachmentPath != "" {
if localImageAttachmentPath != "" {
// Create a multimodal message with both text and image
newMsg = models.NewMultimodalMsg(role, []interface{}{})
// Add the text content
newMsg.AddTextPart(msg)
// Add the image content
imageURL, err := models.CreateImageURLFromPath(imageAttachmentPath)
imageURL, err := models.CreateImageURLFromPath(localImageAttachmentPath)
if err != nil {
logger.Error("failed to create image URL from path", "error", err, "path", imageAttachmentPath)
logger.Error("failed to create image URL from path", "error", err, "path", localImageAttachmentPath)
// If image processing fails, fall back to simple text message
newMsg = models.NewRoleMsg(role, msg)
imageAttachmentPath = "" // Clear the attachment
} else {
newMsg.AddImagePart(imageURL)
// Only clear the global image attachment after successfully processing it in this API call
imageAttachmentPath = "" // Clear the attachment after use
}
} else {
@@ -478,6 +492,11 @@ func (or OpenRouterChat) GetToken() string {
func (or OpenRouterChat) FormMsg(msg, role string, resume bool) (io.Reader, error) {
logger.Debug("formmsg open router completion", "link", cfg.CurrentAPI)
// Capture the image attachment path at the beginning to avoid race conditions
// with API rotation that might clear the global variable
localImageAttachmentPath := imageAttachmentPath
if cfg.ToolUse && !resume {
// prompt += "\n" + cfg.ToolRole + ":\n" + toolSysMsg
// add to chat body
@@ -486,21 +505,36 @@ func (or OpenRouterChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
if msg != "" { // otherwise let the bot continue
var newMsg models.RoleMsg
// Check if we have an image to add to this message
if imageAttachmentPath != "" {
// Create a multimodal message with both text and image
newMsg = models.NewMultimodalMsg(role, []interface{}{})
// Add the text content
newMsg.AddTextPart(msg)
// Add the image content
imageURL, err := models.CreateImageURLFromPath(imageAttachmentPath)
if err != nil {
logger.Error("failed to create image URL from path", "error", err, "path", imageAttachmentPath)
// If image processing fails, fall back to simple text message
logger.Debug("checking for image attachment", "imageAttachmentPath", localImageAttachmentPath, "msg", msg, "role", role)
if localImageAttachmentPath != "" {
logger.Info("processing image attachment for OpenRouter", "path", localImageAttachmentPath, "msg", msg)
// Check if file exists before attempting to create image URL
if _, err := os.Stat(localImageAttachmentPath); os.IsNotExist(err) {
logger.Error("image file does not exist", "path", localImageAttachmentPath)
// Fallback to simple text message
newMsg = models.NewRoleMsg(role, msg)
} else if err != nil {
logger.Error("error checking image file", "path", localImageAttachmentPath, "error", err)
// Fallback to simple text message
newMsg = models.NewRoleMsg(role, msg)
imageAttachmentPath = "" // Clear the attachment
} else {
newMsg.AddImagePart(imageURL)
imageAttachmentPath = "" // Clear the attachment after use
logger.Debug("image file exists, proceeding to create URL", "path", localImageAttachmentPath)
// Create a multimodal message with both text and image
newMsg = models.NewMultimodalMsg(role, []interface{}{})
// Add the text content
newMsg.AddTextPart(msg)
// Add the image content
imageURL, err := models.CreateImageURLFromPath(localImageAttachmentPath)
if err != nil {
logger.Error("failed to create image URL from path", "error", err, "path", localImageAttachmentPath)
// If image processing fails, fall back to simple text message
newMsg = models.NewRoleMsg(role, msg)
} else {
logger.Info("image URL created successfully for OpenRouter", "imageURL", imageURL[:min(len(imageURL), 50)]+"...")
newMsg.AddImagePart(imageURL)
// Only clear the global image attachment after successfully processing it in this API call
imageAttachmentPath = "" // Clear the attachment after use
}
}
} else {
// Create a simple text message
@@ -520,27 +554,118 @@ func (or OpenRouterChat) FormMsg(msg, role string, resume bool) (io.Reader, erro
}
}
// Create copy of chat body with standardized user role
// modifiedBody := *chatBody
bodyCopy := &models.ChatBody{
Messages: make([]models.RoleMsg, len(chatBody.Messages)),
Model: chatBody.Model,
Stream: chatBody.Stream,
}
// modifiedBody.Messages = make([]models.RoleMsg, len(chatBody.Messages))
for i, msg := range chatBody.Messages {
logger.Debug("checking roles", "#", i, "role", msg.Role)
if msg.Role == cfg.UserRole || i == 1 {
// Check if this message has content parts (multimodal) by attempting to marshal and checking structure
msgBytes, err := json.Marshal(msg)
if err != nil {
logger.Error("failed to serialize message for inspection", "error", err)
// Fallback to direct assignment
bodyCopy.Messages[i] = msg
} else {
// Try to deserialize to check if it has content parts
var tempMsg map[string]interface{}
if err := json.Unmarshal(msgBytes, &tempMsg); err != nil {
logger.Error("failed to inspect message structure", "error", err)
bodyCopy.Messages[i] = msg
} else {
// Check if content is an array (indicating content parts) or string (simple content)
if content, ok := tempMsg["content"]; ok {
if _, isArray := content.([]interface{}); isArray {
logger.Info("multimodal message detected", "#", i, "role", msg.Role)
// Deserialize to RoleMsg to access ContentParts
var detailedMsg models.RoleMsg
if err := json.Unmarshal(msgBytes, &detailedMsg); err == nil {
if len(detailedMsg.ContentParts) > 0 {
for j, part := range detailedMsg.ContentParts {
if textPart, ok := part.(models.TextContentPart); ok {
logger.Debug("text content part", "msg#", i, "part#", j, "text", textPart.Text)
} else if imgPart, ok := part.(models.ImageContentPart); ok {
logger.Info("image content part", "msg#", i, "part#", j, "url", imgPart.ImageURL.URL[:min(len(imgPart.ImageURL.URL), 50)]+"...")
} else {
logger.Debug("other content part", "msg#", i, "part#", j, "type", fmt.Sprintf("%T", part))
}
}
}
}
}
}
}
}
// Create a proper copy of the message that preserves all internal state
// First, serialize and deserialize to ensure content parts are preserved
copyMsgBytes, err := json.Marshal(msg)
if err != nil {
logger.Error("failed to serialize message", "error", err)
// Fallback to direct assignment
bodyCopy.Messages[i] = msg
} else {
// Deserialize back to preserve all internal state
var copiedMsg models.RoleMsg
err := json.Unmarshal(copyMsgBytes, &copiedMsg)
if err != nil {
logger.Error("failed to deserialize message", "error", err)
// Fallback to direct assignment
bodyCopy.Messages[i] = msg
} else {
bodyCopy.Messages[i] = copiedMsg
}
}
// Standardize role if it's a user role or first message
if bodyCopy.Messages[i].Role == cfg.UserRole || i == 1 {
bodyCopy.Messages[i].Role = "user"
logger.Debug("replaced role in body", "#", i)
} else {
bodyCopy.Messages[i] = msg
}
}
// Log the final request body before sending to OpenRouter
orBody := models.NewOpenRouterChatReq(*bodyCopy, defaultLCPProps)
data, err := json.Marshal(orBody)
if err != nil {
logger.Error("failed to form a msg", "error", err)
return nil, err
}
logger.Info("OpenRouter request prepared", "messages_count", len(orBody.Messages))
for i, msg := range orBody.Messages {
// Check if this final message has content parts (multimodal)
msgBytes, err := json.Marshal(msg)
if err == nil {
var tempMsg map[string]interface{}
if err := json.Unmarshal(msgBytes, &tempMsg); err == nil {
if content, ok := tempMsg["content"]; ok {
if _, isArray := content.([]interface{}); isArray {
logger.Debug("final message", "#", i, "role", msg.Role, "hasContentParts", true)
// Deserialize to access content parts
var detailedMsg models.RoleMsg
if err := json.Unmarshal(msgBytes, &detailedMsg); err == nil {
if len(detailedMsg.ContentParts) > 0 {
for j, part := range detailedMsg.ContentParts {
if textPart, ok := part.(models.TextContentPart); ok {
logger.Debug("final text part", "msg#", i, "part#", j, "text", textPart.Text)
} else if imgPart, ok := part.(models.ImageContentPart); ok {
logger.Info("final image part sent to OpenRouter", "msg#", i, "part#", j, "url", imgPart.ImageURL.URL[:min(len(imgPart.ImageURL.URL), 50)]+"...")
} else {
logger.Debug("final other part", "msg#", i, "part#", j, "type", fmt.Sprintf("%T", part))
}
}
}
}
} else {
logger.Debug("final message", "#", i, "role", msg.Role, "hasContentParts", false)
}
}
}
}
}
return bytes.NewReader(data), nil
}