Support for Openrouter (#1628)

This commit is contained in:
Dev Khant
2024-08-03 22:51:03 +05:30
committed by GitHub
parent 5837991e5c
commit 04b4807145
4 changed files with 68 additions and 7 deletions

View File

@@ -14,7 +14,7 @@ class OllamaLLM(LLMBase):
if not self.config.model:
self.config.model="llama3.1:70b"
self.client = Client(host=self.config.base_url)
self.client = Client(host=self.config.ollama_base_url)
self._ensure_model_exists()
def _ensure_model_exists(self):

View File

@@ -1,3 +1,4 @@
import os
import json
from typing import Dict, List, Optional
@@ -12,7 +13,11 @@ class OpenAILLM(LLMBase):
if not self.config.model:
self.config.model="gpt-4o"
self.client = OpenAI()
if os.environ.get("OPENROUTER_API_KEY"):
self.client = OpenAI(api_key=os.environ.get("OPENROUTER_API_KEY"), base_url=self.config.openrouter_base_url)
else:
self.client = OpenAI()
def _parse_response(self, response, tools):
"""
@@ -68,6 +73,23 @@ class OpenAILLM(LLMBase):
"max_tokens": self.config.max_tokens,
"top_p": self.config.top_p
}
if os.getenv("OPENROUTER_API_KEY"):
openrouter_params = {}
if self.config.models:
openrouter_params["models"] = self.config.models
openrouter_params["route"] = self.config.route
params.pop("model")
if self.config.site_url and self.config.app_name:
extra_headers={
"HTTP-Referer": self.config.site_url,
"X-Title": self.config.app_name
}
openrouter_params["extra_headers"] = extra_headers
params.update(**openrouter_params)
if response_format:
params["response_format"] = response_format
if tools: