Feat: read img tool for chat endpoint

This commit is contained in:
Grail Finder
2026-03-02 07:12:28 +03:00
parent 742f1ca838
commit caac1d397a
4 changed files with 113 additions and 11 deletions

View File

@@ -469,6 +469,43 @@ func fileRead(args map[string]string) []byte {
return jsonResult
}
func fileReadImage(args map[string]string) []byte {
path, ok := args["path"]
if !ok || path == "" {
msg := "path not provided to file_read_image tool"
logger.Error(msg)
return []byte(msg)
}
path = resolvePath(path)
dataURL, err := models.CreateImageURLFromPath(path)
if err != nil {
msg := "failed to read image; error: " + err.Error()
logger.Error(msg)
return []byte(msg)
}
// result := map[string]any{
// "type": "multimodal_content",
// "parts": []map[string]string{
// {"type": "text", "text": "Image at " + path},
// {"type": "image_url", "url": dataURL},
// },
// }
result := models.MultimodalToolResp{
Type: "multimodal_content",
Parts: []map[string]string{
{"type": "text", "text": "Image at " + path},
{"type": "image_url", "url": dataURL},
},
}
jsonResult, err := json.Marshal(result)
if err != nil {
msg := "failed to marshal result; error: " + err.Error()
logger.Error(msg)
return []byte(msg)
}
return jsonResult
}
func fileWrite(args map[string]string) []byte {
path, ok := args["path"]
if !ok || path == "" {
@@ -1101,6 +1138,7 @@ var fnMap = map[string]fnSig{
"read_url_raw": readURLRaw,
"file_create": fileCreate,
"file_read": fileRead,
"file_read_image": fileReadImage,
"file_write": fileWrite,
"file_write_append": fileWriteAppend,
"file_edit": fileEdit,
@@ -1327,6 +1365,24 @@ var baseTools = []models.Tool{
},
},
},
// file_read_image
models.Tool{
Type: "function",
Function: models.ToolFunc{
Name: "file_read_image",
Description: "Read an image file and return it for multimodal LLM viewing. Supports png, jpg, jpeg, gif, webp formats. Use when you need the LLM to see and analyze an image.",
Parameters: models.ToolFuncParams{
Type: "object",
Required: []string{"path"},
Properties: map[string]models.ToolArgProps{
"path": models.ToolArgProps{
Type: "string",
Description: "path of the image file to read",
},
},
},
},
},
// file_write
models.Tool{
Type: "function",