Feat: llm call retries and model switch

llmapi/main.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"time"
 )
 
 var (
@@ -427,27 +428,64 @@ func (b *Bot) BuildPrompt(room *models.Room) string {
 
 func (b *Bot) CallLLM(prompt string) ([]byte, error) {
 	method := "POST"
-	payload := b.LLMParser.MakePayload(prompt)
+	// Generate the payload once as bytes
+	payloadReader := b.LLMParser.MakePayload(prompt)
 	client := &http.Client{}
-	req, err := http.NewRequest(method, b.cfg.LLMConfig.URL, payload)
-	if err != nil {
-		b.log.Error("failed to make new request", "error", err, "url", b.cfg.LLMConfig.URL)
-		return nil, err
+	maxRetries := 6
+	baseDelay := 2 // seconds
+	for attempt := 0; attempt < maxRetries; attempt++ {
+		// Create a new request for the attempt
+		req, err := http.NewRequest(method, b.cfg.LLMConfig.URL, payloadReader)
+		if err != nil {
+			if attempt == maxRetries-1 {
+				return nil, fmt.Errorf("failed to create request: %w", err)
+			}
+			b.log.Error("failed to make new request; will retry", "error", err, "url", b.cfg.LLMConfig.URL, "attempt", attempt)
+			time.Sleep(time.Duration(baseDelay) * time.Second * time.Duration(attempt+1))
+			continue
+		}
+		req.Header.Add("Content-Type", "application/json")
+		req.Header.Add("Accept", "application/json")
+		req.Header.Add("Authorization", "Bearer "+b.cfg.LLMConfig.TOKEN)
+		resp, err := client.Do(req)
+		if err != nil {
+			if attempt == maxRetries-1 {
+				return nil, fmt.Errorf("http request failed: %w", err)
+			}
+			b.log.Error("http request failed; will retry", "error", err, "url", b.cfg.LLMConfig.URL, "attempt", attempt)
+			delay := time.Duration(baseDelay*(attempt+1)) * time.Second
+			time.Sleep(delay)
+			continue
+		}
+		body, err := io.ReadAll(resp.Body)
+		resp.Body.Close()
+		if err != nil {
+			if attempt == maxRetries-1 {
+				return nil, fmt.Errorf("failed to read response body: %w", err)
+			}
+			b.log.Error("failed to read response body; will retry", "error", err, "url", b.cfg.LLMConfig.URL, "attempt", attempt)
+			delay := time.Duration(baseDelay*(attempt+1)) * time.Second
+			time.Sleep(delay)
+			continue
+		}
+		// Check status code
+		if resp.StatusCode >= 400 && resp.StatusCode < 600 {
+			if attempt == maxRetries-1 {
+				return nil, fmt.Errorf("after %d retries, still got status %d", maxRetries, resp.StatusCode)
+			}
+			b.log.Warn("retriable status code; will retry", "code", resp.StatusCode, "attempt", attempt)
+			delay := time.Duration((baseDelay * (1 << attempt))) * time.Second
+			time.Sleep(delay)
+			continue
+		}
+		if resp.StatusCode != http.StatusOK {
+			// For non-retriable errors, return immediately
+			return nil, fmt.Errorf("non-retriable status %d, body: %s", resp.StatusCode, string(body))
+		}
+		// Success
+		b.log.Debug("llm resp", "body", string(body), "url", b.cfg.LLMConfig.URL, "attempt", attempt)
+		return body, nil
 	}
-	req.Header.Add("Content-Type", "application/json")
-	req.Header.Add("Accept", "application/json")
-	req.Header.Add("Authorization", "Bearer "+b.cfg.LLMConfig.TOKEN)
-	res, err := client.Do(req)
-	if err != nil {
-		b.log.Error("failed to make request", "error", err, "url", b.cfg.LLMConfig.URL)
-		return nil, err
-	}
-	defer res.Body.Close()
-	body, err := io.ReadAll(res.Body)
-	if err != nil {
-		b.log.Error("failed to read resp body", "error", err, "url", b.cfg.LLMConfig.URL)
-		return nil, err
-	}
-	b.log.Debug("llm resp", "body", string(body), "url", b.cfg.LLMConfig.URL)
-	return body, nil
+	// This line should not be reached because each error path returns in the loop.
+	return nil, fmt.Errorf("unknown error in retry loop")
 }