Feature/vllm support (#2981)

mem0/configs/llms/base.py

@@ -44,6 +44,8 @@ class BaseLlmConfig(ABC):
         # LM Studio specific
         lmstudio_base_url: Optional[str] = "http://localhost:1234/v1",
         lmstudio_response_format: dict = None,
+        # vLLM specific
+        vllm_base_url: Optional[str] = "http://localhost:8000/v1",
         # AWS Bedrock specific
         aws_access_key_id: Optional[str] = None,
         aws_secret_access_key: Optional[str] = None,
@@ -98,6 +100,8 @@ class BaseLlmConfig(ABC):
         :type lmstudio_base_url: Optional[str], optional
         :param lmstudio_response_format: LM Studio response format to be use, defaults to None
         :type lmstudio_response_format: Optional[Dict], optional
+        :param vllm_base_url: vLLM base URL to be use, defaults to "http://localhost:8000/v1"
+        :type vllm_base_url: Optional[str], optional
         """
 
         self.model = model
@@ -139,6 +143,9 @@ class BaseLlmConfig(ABC):
         self.lmstudio_base_url = lmstudio_base_url
         self.lmstudio_response_format = lmstudio_response_format
 
+        # vLLM specific
+        self.vllm_base_url = vllm_base_url
+
         # AWS Bedrock specific
         self.aws_access_key_id = aws_access_key_id
         self.aws_secret_access_key = aws_secret_access_key

mem0/llms/configs.py

@@ -26,6 +26,7 @@ class LlmConfig(BaseModel):
             "xai",
             "sarvam",
             "lmstudio",
+            "vllm",
             "langchain",
         ):
             return v

mem0/llms/vllm.py

@@ -0,0 +1,84 @@
+import json
+import os
+from typing import Dict, List, Optional
+
+from mem0.configs.llms.base import BaseLlmConfig
+from mem0.llms.base import LLMBase
+
+
+class VllmLLM(LLMBase):
+    def __init__(self, config: Optional[BaseLlmConfig] = None):
+        super().__init__(config)
+
+        if not self.config.model:
+            self.config.model = "Qwen/Qwen2.5-32B-Instruct"
+
+        self.config.api_key = self.config.api_key or os.getenv("VLLM_API_KEY") or "vllm-api-key"
+        base_url = self.config.vllm_base_url or os.getenv("VLLM_BASE_URL")
+
+        self.client = OpenAI(base_url=base_url, api_key=self.config.api_key)
+
+    def _parse_response(self, response, tools):
+        """
+        Process the response based on whether tools are used or not.
+
+        Args:
+            response: The raw response from API.
+            tools: The list of tools provided in the request.
+
+        Returns:
+            str or dict: The processed response.
+        """
+        if tools:
+            processed_response = {
+                "content": response.choices[0].message.content,
+                "tool_calls": [],
+            }
+
+            if response.choices[0].message.tool_calls:
+                for tool_call in response.choices[0].message.tool_calls:
+                    processed_response["tool_calls"].append({
+                        "name": tool_call.function.name,
+                        "arguments": json.loads(tool_call.function.arguments),
+                    })
+
+            return processed_response
+        else:
+            return response.choices[0].message.content
+
+    def generate_response(
+        self,
+        messages: List[Dict[str, str]],
+        response_format=None,
+        tools: Optional[List[Dict]] = None,
+        tool_choice: str = "auto",
+    ):
+        """
+        Generate a response based on the given messages using vLLM.
+
+        Args:
+            messages (list): List of message dicts containing 'role' and 'content'.
+            response_format (str or object, optional): Format of the response. Defaults to "text".
+            tools (list, optional): List of tools that the model can call. Defaults to None.
+            tool_choice (str, optional): Tool choice method. Defaults to "auto".
+
+        Returns:
+            str: The generated response.
+        """
+        params = {
+            "model": self.config.model,
+            "messages": messages,
+            "temperature": self.config.temperature,
+            "max_tokens": self.config.max_tokens,
+            "top_p": self.config.top_p,
+        }
+
+        if response_format:
+            params["response_format"] = response_format
+
+        if tools:
+            params["tools"] = tools
+            params["tool_choice"] = tool_choice
+
+        response = self.client.chat.completions.create(**params)
+        return self._parse_response(response, tools)

mem0/utils/factory.py

@@ -29,6 +29,7 @@ class LlmFactory:
         "xai": "mem0.llms.xai.XAILLM",
         "sarvam": "mem0.llms.sarvam.SarvamLLM",
         "lmstudio": "mem0.llms.lmstudio.LMStudioLLM",
+        "vllm": "mem0.llms.vllm.VllmLLM",
         "langchain": "mem0.llms.langchain.LangchainLLM",
     }