diff --git a/examples/multimodal-demo/.gitattributes b/examples/multimodal-demo/.gitattributes new file mode 100644 index 00000000..dfe07704 --- /dev/null +++ b/examples/multimodal-demo/.gitattributes @@ -0,0 +1,2 @@ +# Auto detect text files and perform LF normalization +* text=auto diff --git a/examples/multimodal-demo/.gitignore b/examples/multimodal-demo/.gitignore new file mode 100644 index 00000000..9767597e --- /dev/null +++ b/examples/multimodal-demo/.gitignore @@ -0,0 +1,29 @@ +**/.env +**/node_modules +**/dist +**/.DS_Store + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/examples/multimodal-demo/components.json b/examples/multimodal-demo/components.json new file mode 100644 index 00000000..eaf9959b --- /dev/null +++ b/examples/multimodal-demo/components.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://ui.shadcn.com/schema.json", + "style": "new-york", + "rsc": false, + "tsx": true, + "tailwind": { + "config": "tailwind.config.js", + "css": "src/index.css", + "baseColor": "zinc", + "cssVariables": true, + "prefix": "" + }, + "aliases": { + "components": "@/components", + "utils": "@/libs/utils", + "ui": "@/components/ui", + "lib": "@/libs", + "hooks": "@/hooks" + } +} \ No newline at end of file diff --git a/examples/multimodal-demo/eslint.config.js b/examples/multimodal-demo/eslint.config.js new file mode 100644 index 00000000..092408a9 --- /dev/null +++ b/examples/multimodal-demo/eslint.config.js @@ -0,0 +1,28 @@ +import js from '@eslint/js' +import globals from 'globals' +import reactHooks from 'eslint-plugin-react-hooks' +import reactRefresh from 'eslint-plugin-react-refresh' +import tseslint from 'typescript-eslint' + +export default tseslint.config( + { ignores: ['dist'] }, + { + extends: [js.configs.recommended, ...tseslint.configs.recommended], + files: ['**/*.{ts,tsx}'], + languageOptions: { + ecmaVersion: 2020, + globals: globals.browser, + }, + plugins: { + 'react-hooks': reactHooks, + 'react-refresh': reactRefresh, + }, + rules: { + ...reactHooks.configs.recommended.rules, + 'react-refresh/only-export-components': [ + 'warn', + { allowConstantExport: true }, + ], + }, + }, +) diff --git a/examples/multimodal-demo/index.html b/examples/multimodal-demo/index.html new file mode 100644 index 00000000..e2135b1c --- /dev/null +++ b/examples/multimodal-demo/index.html @@ -0,0 +1,13 @@ + + + + + + + JustChat | Chat with AI + + +
+ + + diff --git a/examples/vercel-ai-sdk-chat-app/package.json b/examples/multimodal-demo/package.json similarity index 85% rename from examples/vercel-ai-sdk-chat-app/package.json rename to examples/multimodal-demo/package.json index 47a3e4c4..3a7a7434 100644 --- a/examples/vercel-ai-sdk-chat-app/package.json +++ b/examples/multimodal-demo/package.json @@ -24,9 +24,11 @@ "clsx": "^2.1.1", "framer-motion": "^11.11.11", "lucide-react": "^0.454.0", + "openai": "^4.86.2", "react": "^18.3.1", "react-dom": "^18.3.1", "react-markdown": "^9.0.1", + "mem0ai": "2.1.2", "tailwind-merge": "^2.5.4", "tailwindcss-animate": "^1.0.7", "zod": "^3.23.8" @@ -46,6 +48,7 @@ "tailwindcss": "^3.4.14", "typescript": "~5.6.2", "typescript-eslint": "^8.11.0", - "vite": "^5.4.10" - } + "vite": "^6.2.1" + }, + "packageManager": "pnpm@10.5.2+sha512.da9dc28cd3ff40d0592188235ab25d3202add8a207afbedc682220e4a0029ffbff4562102b9e6e46b4e3f9e8bd53e6d05de48544b0c57d4b0179e22c76d1199b" } diff --git a/examples/multimodal-demo/postcss.config.js b/examples/multimodal-demo/postcss.config.js new file mode 100644 index 00000000..2e7af2b7 --- /dev/null +++ b/examples/multimodal-demo/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +} diff --git a/examples/multimodal-demo/public/mem0_logo.jpeg b/examples/multimodal-demo/public/mem0_logo.jpeg new file mode 100644 index 00000000..eb02b0ec Binary files /dev/null and b/examples/multimodal-demo/public/mem0_logo.jpeg differ diff --git a/examples/multimodal-demo/src/App.tsx b/examples/multimodal-demo/src/App.tsx new file mode 100644 index 00000000..4564ce5d --- /dev/null +++ b/examples/multimodal-demo/src/App.tsx @@ -0,0 +1,13 @@ +import Home from "./page" + + +function App() { + + return ( + <> + + + ) +} + +export default App diff --git a/examples/multimodal-demo/src/assets/mem0_logo.jpeg b/examples/multimodal-demo/src/assets/mem0_logo.jpeg new file mode 100644 index 00000000..eb02b0ec Binary files /dev/null and b/examples/multimodal-demo/src/assets/mem0_logo.jpeg differ diff --git a/examples/multimodal-demo/src/assets/react.svg b/examples/multimodal-demo/src/assets/react.svg new file mode 100644 index 00000000..6c87de9b --- /dev/null +++ b/examples/multimodal-demo/src/assets/react.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/examples/multimodal-demo/src/assets/user.jpg b/examples/multimodal-demo/src/assets/user.jpg new file mode 100644 index 00000000..f2e7fc22 Binary files /dev/null and b/examples/multimodal-demo/src/assets/user.jpg differ diff --git a/examples/multimodal-demo/src/components/api-settings-popup.tsx b/examples/multimodal-demo/src/components/api-settings-popup.tsx new file mode 100644 index 00000000..8a4eac39 --- /dev/null +++ b/examples/multimodal-demo/src/components/api-settings-popup.tsx @@ -0,0 +1,91 @@ +import { Dispatch, SetStateAction, useContext, useEffect, useState } from 'react' +import { Button } from "@/components/ui/button" +import { Input } from "@/components/ui/input" +import { Label } from "@/components/ui/label" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" +import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from "@/components/ui/dialog" +import GlobalContext from '@/contexts/GlobalContext' +import { Provider } from '@/constants/messages' +export default function ApiSettingsPopup(props: { isOpen: boolean, setIsOpen: Dispatch> }) { + const {isOpen, setIsOpen} = props + const [mem0ApiKey, setMem0ApiKey] = useState('') + const [providerApiKey, setProviderApiKey] = useState('') + const [provider, setProvider] = useState('OpenAI') + const { selectorHandler, selectedOpenAIKey, selectedMem0Key, selectedProvider } = useContext(GlobalContext); + + const handleSave = () => { + // Here you would typically save the settings to your backend or local storage + selectorHandler(mem0ApiKey, providerApiKey, provider as Provider); + setIsOpen(false) + } + + useEffect(() => { + if (selectedOpenAIKey) { + setProviderApiKey(selectedOpenAIKey); + } + if (selectedMem0Key) { + setMem0ApiKey(selectedMem0Key); + } + if (selectedProvider) { + setProvider(selectedProvider); + } + }, [selectedOpenAIKey, selectedMem0Key, selectedProvider]); + + + + return ( + <> + + + + API Configuration Settings + +
+
+ + setMem0ApiKey(e.target.value)} + className="col-span-3 rounded-3xl" + /> +
+
+ + setProviderApiKey(e.target.value)} + className="col-span-3 rounded-3xl" + /> +
+
+ + +
+
+ + + + + + + + ) +} \ No newline at end of file diff --git a/examples/multimodal-demo/src/components/chevron-toggle.tsx b/examples/multimodal-demo/src/components/chevron-toggle.tsx new file mode 100644 index 00000000..7b8b128e --- /dev/null +++ b/examples/multimodal-demo/src/components/chevron-toggle.tsx @@ -0,0 +1,35 @@ +import { Button } from "@/components/ui/button"; +import { ChevronLeft, ChevronRight } from "lucide-react"; +import React from "react"; + +const ChevronToggle = (props: { + isMemoriesExpanded: boolean; + setIsMemoriesExpanded: React.Dispatch>; +}) => { + const { isMemoriesExpanded, setIsMemoriesExpanded } = props; + return ( + <> +
+
+ +
+
+ + ); +}; + +export default ChevronToggle; diff --git a/examples/multimodal-demo/src/components/header.tsx b/examples/multimodal-demo/src/components/header.tsx new file mode 100644 index 00000000..7ddbd37d --- /dev/null +++ b/examples/multimodal-demo/src/components/header.tsx @@ -0,0 +1,81 @@ +import { Button } from "@/components/ui/button"; +import { ChevronRight, X, RefreshCcw, Settings } from "lucide-react"; +import { Dispatch, SetStateAction, useContext, useEffect, useState } from "react"; +import GlobalContext from "../contexts/GlobalContext"; +import { Input } from "./ui/input"; + +const Header = (props: { + setIsSettingsOpen: Dispatch>; +}) => { + const { setIsSettingsOpen } = props; + const { selectUserHandler, clearUserHandler, selectedUser, clearConfiguration } = useContext(GlobalContext); + const [userId, setUserId] = useState(""); + + const handleSelectUser = (e: React.ChangeEvent) => { + setUserId(e.target.value); + }; + + const handleClearUser = () => { + clearUserHandler(); + setUserId(""); + }; + + const handleSubmit = () => { + selectUserHandler(userId); + }; + + // New function to handle key down events + const handleKeyDown = (e: React.KeyboardEvent) => { + if (e.key === 'Enter') { + e.preventDefault(); // Prevent form submission if it's in a form + handleSubmit(); + } + }; + + useEffect(() => { + if (selectedUser) { + setUserId(selectedUser); + } + }, [selectedUser]); + + return ( + <> +
+
+ Mem0 Assistant +
+
+
+ + + +
+
+ + +
+
+
+ + ); +}; + +export default Header; diff --git a/examples/multimodal-demo/src/components/input-area.tsx b/examples/multimodal-demo/src/components/input-area.tsx new file mode 100644 index 00000000..877e19a2 --- /dev/null +++ b/examples/multimodal-demo/src/components/input-area.tsx @@ -0,0 +1,107 @@ +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import GlobalContext from "@/contexts/GlobalContext"; +import { FileInfo } from "@/types"; +import { Images, Send, X } from "lucide-react"; +import { useContext, useRef, useState } from "react"; + +const InputArea = () => { + const [inputValue, setInputValue] = useState(""); + const { handleSend, selectedFile, setSelectedFile, setFile } = useContext(GlobalContext); + const [loading, setLoading] = useState(false); + + const ref = useRef(null); + const fileInputRef = useRef(null) + + const handleFileChange = (event: React.ChangeEvent) => { + const file = event.target.files?.[0] + if (file) { + setSelectedFile({ + name: file.name, + type: file.type, + size: file.size + }) + setFile(file) + } + } + + const handleSendController = async () => { + setLoading(true); + setInputValue(""); + await handleSend(inputValue); + setLoading(false); + + // focus on input + setTimeout(() => { + ref.current?.focus(); + }, 0); + }; + + const handleClosePopup = () => { + setSelectedFile(null) + if (fileInputRef.current) { + fileInputRef.current.value = '' + } + } + + return ( + <> +
+
+
+
+ + + {selectedFile && } +
+
+ setInputValue(e.target.value)} + onKeyDown={(e) => e.key === "Enter" && handleSendController()} + placeholder="Type a message..." + className="flex-1 pl-10 rounded-3xl" + disabled={loading} + ref={ref} + /> +
+ +
+
+
+ + ); +}; + +const FileInfoPopup = ({ file, onClose }: { file: FileInfo, onClose: () => void }) => { + return ( +
+
+
+

{file.name}

+ +
+

Type: {file.type}

+

Size: {(file.size / 1024).toFixed(2)} KB

+
+
+ ) +} + +export default InputArea; diff --git a/examples/multimodal-demo/src/components/memories.tsx b/examples/multimodal-demo/src/components/memories.tsx new file mode 100644 index 00000000..940fbe63 --- /dev/null +++ b/examples/multimodal-demo/src/components/memories.tsx @@ -0,0 +1,84 @@ +import { Badge } from "@/components/ui/badge"; +import { Card } from "@/components/ui/card"; +import { ScrollArea } from "@radix-ui/react-scroll-area"; +import { Memory } from "../types"; +import GlobalContext from "@/contexts/GlobalContext"; +import { useContext } from "react"; +import { motion } from "framer-motion"; + + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const MemoryItem = ({ memory }: { memory: Memory; index: number }) => { + return ( + +
+

{memory.content}

+
+
+ {new Date(memory.timestamp).toLocaleString()} +
+
+ {memory.tags.map((tag) => ( + + {tag} + + ))} +
+ + ); +}; + +const Memories = (props: { isMemoriesExpanded: boolean }) => { + const { isMemoriesExpanded } = props; + const { memories } = useContext(GlobalContext); + + return ( + +
+ + Relevant Memories ({memories.length}) + +
+ {memories.length === 0 && ( + + No relevant memories found. +
+ Only the relevant memories will be displayed here. + + )} + + + {/* */} + {memories.map((memory: Memory, index: number) => ( + + ))} + {/* */} + + + + ); +}; + +export default Memories; \ No newline at end of file diff --git a/examples/multimodal-demo/src/components/messages.tsx b/examples/multimodal-demo/src/components/messages.tsx new file mode 100644 index 00000000..38e5a59e --- /dev/null +++ b/examples/multimodal-demo/src/components/messages.tsx @@ -0,0 +1,102 @@ +import { Avatar, AvatarFallback, AvatarImage } from "@/components/ui/avatar"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { Message } from "../types"; +import { useContext, useEffect, useRef } from "react"; +import GlobalContext from "@/contexts/GlobalContext"; +import Markdown from "react-markdown"; +import Mem00Logo from "../assets/mem0_logo.jpeg"; +import UserLogo from "../assets/user.jpg"; + +const Messages = () => { + const { messages, thinking } = useContext(GlobalContext); + const scrollAreaRef = useRef(null); + + // scroll to bottom + useEffect(() => { + if (scrollAreaRef.current) { + scrollAreaRef.current.scrollTop += 40; // Scroll down by 40 pixels + } + }, [messages, thinking]); + + return ( + <> + +
+ {messages.map((message: Message) => ( +
+
+
+ + + + {message.sender === "assistant" ? "AI" : "U"} + + +
+
+ {message.image && ( +
+ Message attachment +
+ )} + {message.content} + + {message.timestamp} + +
+
+
+ ))} + {thinking && ( +
+
+ + + {"AI"} + +
+
+
+
+
+
+
+
+
+ )} +
+ + + ); +}; + +export default Messages; diff --git a/examples/multimodal-demo/src/components/ui/avatar.tsx b/examples/multimodal-demo/src/components/ui/avatar.tsx new file mode 100644 index 00000000..9065241a --- /dev/null +++ b/examples/multimodal-demo/src/components/ui/avatar.tsx @@ -0,0 +1,50 @@ +"use client" + +import * as React from "react" +import * as AvatarPrimitive from "@radix-ui/react-avatar" + +import { cn } from "@/libs/utils" + +const Avatar = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +Avatar.displayName = AvatarPrimitive.Root.displayName + +const AvatarImage = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +AvatarImage.displayName = AvatarPrimitive.Image.displayName + +const AvatarFallback = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +AvatarFallback.displayName = AvatarPrimitive.Fallback.displayName + +export { Avatar, AvatarImage, AvatarFallback } diff --git a/examples/multimodal-demo/src/components/ui/badge.tsx b/examples/multimodal-demo/src/components/ui/badge.tsx new file mode 100644 index 00000000..060b2f11 --- /dev/null +++ b/examples/multimodal-demo/src/components/ui/badge.tsx @@ -0,0 +1,36 @@ +import * as React from "react" +import { cva, type VariantProps } from "class-variance-authority" + +import { cn } from "@/libs/utils" + +const badgeVariants = cva( + "inline-flex items-center rounded-md border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2", + { + variants: { + variant: { + default: + "border-transparent bg-primary text-primary-foreground shadow hover:bg-primary/80", + secondary: + "border-transparent bg-secondary text-secondary-foreground hover:bg-secondary/80", + destructive: + "border-transparent bg-destructive text-destructive-foreground shadow hover:bg-destructive/80", + outline: "text-foreground", + }, + }, + defaultVariants: { + variant: "default", + }, + } +) + +export interface BadgeProps + extends React.HTMLAttributes, + VariantProps {} + +function Badge({ className, variant, ...props }: BadgeProps) { + return ( +
+ ) +} + +export { Badge, badgeVariants } diff --git a/examples/multimodal-demo/src/components/ui/button.tsx b/examples/multimodal-demo/src/components/ui/button.tsx new file mode 100644 index 00000000..3e85ff7a --- /dev/null +++ b/examples/multimodal-demo/src/components/ui/button.tsx @@ -0,0 +1,57 @@ +import * as React from "react" +import { Slot } from "@radix-ui/react-slot" +import { cva, type VariantProps } from "class-variance-authority" + +import { cn } from "@/libs/utils" + +const buttonVariants = cva( + "inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0", + { + variants: { + variant: { + default: + "bg-primary text-primary-foreground shadow hover:bg-primary/90", + destructive: + "bg-destructive text-destructive-foreground shadow-sm hover:bg-destructive/90", + outline: + "border border-input bg-background shadow-sm hover:bg-accent hover:text-accent-foreground", + secondary: + "bg-secondary text-secondary-foreground shadow-sm hover:bg-secondary/80", + ghost: "hover:bg-accent hover:text-accent-foreground", + link: "text-primary underline-offset-4 hover:underline", + }, + size: { + default: "h-9 px-4 py-2", + sm: "h-8 rounded-md px-3 text-xs", + lg: "h-10 rounded-md px-8", + icon: "h-9 w-9", + }, + }, + defaultVariants: { + variant: "default", + size: "default", + }, + } +) + +export interface ButtonProps + extends React.ButtonHTMLAttributes, + VariantProps { + asChild?: boolean +} + +const Button = React.forwardRef( + ({ className, variant, size, asChild = false, ...props }, ref) => { + const Comp = asChild ? Slot : "button" + return ( + + ) + } +) +Button.displayName = "Button" + +export { Button, buttonVariants } diff --git a/examples/multimodal-demo/src/components/ui/card.tsx b/examples/multimodal-demo/src/components/ui/card.tsx new file mode 100644 index 00000000..e90617d5 --- /dev/null +++ b/examples/multimodal-demo/src/components/ui/card.tsx @@ -0,0 +1,76 @@ +import * as React from "react" + +import { cn } from "@/libs/utils" + +const Card = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +Card.displayName = "Card" + +const CardHeader = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +CardHeader.displayName = "CardHeader" + +const CardTitle = React.forwardRef< + HTMLParagraphElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +

+)) +CardTitle.displayName = "CardTitle" + +const CardDescription = React.forwardRef< + HTMLParagraphElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +

+)) +CardDescription.displayName = "CardDescription" + +const CardContent = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +

+)) +CardContent.displayName = "CardContent" + +const CardFooter = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +CardFooter.displayName = "CardFooter" + +export { Card, CardHeader, CardFooter, CardTitle, CardDescription, CardContent } diff --git a/examples/multimodal-demo/src/components/ui/dialog.tsx b/examples/multimodal-demo/src/components/ui/dialog.tsx new file mode 100644 index 00000000..1796099a --- /dev/null +++ b/examples/multimodal-demo/src/components/ui/dialog.tsx @@ -0,0 +1,120 @@ +import * as React from "react" +import * as DialogPrimitive from "@radix-ui/react-dialog" +import { Cross2Icon } from "@radix-ui/react-icons" + +import { cn } from "@/libs/utils" + +const Dialog = DialogPrimitive.Root + +const DialogTrigger = DialogPrimitive.Trigger + +const DialogPortal = DialogPrimitive.Portal + +const DialogClose = DialogPrimitive.Close + +const DialogOverlay = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +DialogOverlay.displayName = DialogPrimitive.Overlay.displayName + +const DialogContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, ...props }, ref) => ( + + + + {children} + + + Close + + + +)) +DialogContent.displayName = DialogPrimitive.Content.displayName + +const DialogHeader = ({ + className, + ...props +}: React.HTMLAttributes) => ( +
+) +DialogHeader.displayName = "DialogHeader" + +const DialogFooter = ({ + className, + ...props +}: React.HTMLAttributes) => ( +
+) +DialogFooter.displayName = "DialogFooter" + +const DialogTitle = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +DialogTitle.displayName = DialogPrimitive.Title.displayName + +const DialogDescription = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +DialogDescription.displayName = DialogPrimitive.Description.displayName + +export { + Dialog, + DialogPortal, + DialogOverlay, + DialogTrigger, + DialogClose, + DialogContent, + DialogHeader, + DialogFooter, + DialogTitle, + DialogDescription, +} diff --git a/examples/multimodal-demo/src/components/ui/input.tsx b/examples/multimodal-demo/src/components/ui/input.tsx new file mode 100644 index 00000000..d2bdc607 --- /dev/null +++ b/examples/multimodal-demo/src/components/ui/input.tsx @@ -0,0 +1,25 @@ +import * as React from "react" + +import { cn } from "@/libs/utils" + +export interface InputProps + extends React.InputHTMLAttributes {} + +const Input = React.forwardRef( + ({ className, type, ...props }, ref) => { + return ( + + ) + } +) +Input.displayName = "Input" + +export { Input } diff --git a/examples/multimodal-demo/src/components/ui/label.tsx b/examples/multimodal-demo/src/components/ui/label.tsx new file mode 100644 index 00000000..4a31cf96 --- /dev/null +++ b/examples/multimodal-demo/src/components/ui/label.tsx @@ -0,0 +1,24 @@ +import * as React from "react" +import * as LabelPrimitive from "@radix-ui/react-label" +import { cva, type VariantProps } from "class-variance-authority" + +import { cn } from "@/libs/utils" + +const labelVariants = cva( + "text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70" +) + +const Label = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef & + VariantProps +>(({ className, ...props }, ref) => ( + +)) +Label.displayName = LabelPrimitive.Root.displayName + +export { Label } diff --git a/examples/multimodal-demo/src/components/ui/scroll-area.tsx b/examples/multimodal-demo/src/components/ui/scroll-area.tsx new file mode 100644 index 00000000..94e4b135 --- /dev/null +++ b/examples/multimodal-demo/src/components/ui/scroll-area.tsx @@ -0,0 +1,46 @@ +import * as React from "react" +import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area" + +import { cn } from "@/libs/utils" + +const ScrollArea = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, ...props }, ref) => ( + + + {children} + + + + +)) +ScrollArea.displayName = ScrollAreaPrimitive.Root.displayName + +const ScrollBar = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, orientation = "vertical", ...props }, ref) => ( + + + +)) +ScrollBar.displayName = ScrollAreaPrimitive.ScrollAreaScrollbar.displayName + +export { ScrollArea, ScrollBar } diff --git a/examples/multimodal-demo/src/components/ui/select.tsx b/examples/multimodal-demo/src/components/ui/select.tsx new file mode 100644 index 00000000..cdf9257b --- /dev/null +++ b/examples/multimodal-demo/src/components/ui/select.tsx @@ -0,0 +1,164 @@ +"use client" + +import * as React from "react" +import { + CaretSortIcon, + CheckIcon, + ChevronDownIcon, + ChevronUpIcon, +} from "@radix-ui/react-icons" +import * as SelectPrimitive from "@radix-ui/react-select" + +import { cn } from "@/libs/utils" + +const Select = SelectPrimitive.Root + +const SelectGroup = SelectPrimitive.Group + +const SelectValue = SelectPrimitive.Value + +const SelectTrigger = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, ...props }, ref) => ( + span]:line-clamp-1", + className + )} + {...props} + > + {children} + + + + +)) +SelectTrigger.displayName = SelectPrimitive.Trigger.displayName + +const SelectScrollUpButton = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + + + +)) +SelectScrollUpButton.displayName = SelectPrimitive.ScrollUpButton.displayName + +const SelectScrollDownButton = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + + + +)) +SelectScrollDownButton.displayName = + SelectPrimitive.ScrollDownButton.displayName + +const SelectContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, position = "popper", ...props }, ref) => ( + + + + + {children} + + + + +)) +SelectContent.displayName = SelectPrimitive.Content.displayName + +const SelectLabel = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +SelectLabel.displayName = SelectPrimitive.Label.displayName + +const SelectItem = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, ...props }, ref) => ( + + + + + + + {children} + +)) +SelectItem.displayName = SelectPrimitive.Item.displayName + +const SelectSeparator = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +SelectSeparator.displayName = SelectPrimitive.Separator.displayName + +export { + Select, + SelectGroup, + SelectValue, + SelectTrigger, + SelectContent, + SelectLabel, + SelectItem, + SelectSeparator, + SelectScrollUpButton, + SelectScrollDownButton, +} diff --git a/examples/multimodal-demo/src/constants/messages.ts b/examples/multimodal-demo/src/constants/messages.ts new file mode 100644 index 00000000..af3280a0 --- /dev/null +++ b/examples/multimodal-demo/src/constants/messages.ts @@ -0,0 +1,31 @@ +import { Message } from "@/types"; + +export const WELCOME_MESSAGE: Message = { + id: "1", + content: "👋 Hi there! I'm your personal assistant. How can I help you today? 😊", + sender: "assistant", + timestamp: new Date().toLocaleTimeString(), +}; + +export const INVALID_CONFIG_MESSAGE: Message = { + id: "2", + content: "Invalid configuration. Please check your API keys, and add a user and try again.", + sender: "assistant", + timestamp: new Date().toLocaleTimeString(), +}; + +export const ERROR_MESSAGE: Message = { + id: "3", + content: "Something went wrong. Please try again.", + sender: "assistant", + timestamp: new Date().toLocaleTimeString(), +}; + +export const AI_MODELS = { + openai: "gpt-4o", + anthropic: "claude-3-haiku-20240307", + cohere: "command-r-plus", + groq: "gemma2-9b-it", +} as const; + +export type Provider = keyof typeof AI_MODELS; \ No newline at end of file diff --git a/examples/multimodal-demo/src/contexts/GlobalContext.tsx b/examples/multimodal-demo/src/contexts/GlobalContext.tsx new file mode 100644 index 00000000..755ea829 --- /dev/null +++ b/examples/multimodal-demo/src/contexts/GlobalContext.tsx @@ -0,0 +1,110 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { createContext } from 'react'; +import { Message, Memory, FileInfo } from '@/types'; +import { useAuth } from '@/hooks/useAuth'; +import { useChat } from '@/hooks/useChat'; +import { useFileHandler } from '@/hooks/useFileHandler'; +import { Provider } from '@/constants/messages'; + +interface GlobalContextType { + selectedUser: string; + selectUserHandler: (user: string) => void; + clearUserHandler: () => void; + messages: Message[]; + memories: Memory[]; + handleSend: (content: string) => Promise; + thinking: boolean; + selectedMem0Key: string; + selectedOpenAIKey: string; + selectedProvider: Provider; + selectorHandler: (mem0: string, openai: string, provider: Provider) => void; + clearConfiguration: () => void; + selectedFile: FileInfo | null; + setSelectedFile: (file: FileInfo | null) => void; + file: File | null; + setFile: (file: File | null) => void; +} + +const GlobalContext = createContext({} as GlobalContextType); + +const GlobalState = (props: { children: React.ReactNode }) => { + const { + mem0ApiKey: selectedMem0Key, + openaiApiKey: selectedOpenAIKey, + provider: selectedProvider, + user: selectedUser, + setAuth: selectorHandler, + setUser: selectUserHandler, + clearAuth: clearConfiguration, + clearUser: clearUserHandler, + } = useAuth(); + + const { + selectedFile, + file, + fileData, + setSelectedFile, + handleFile, + clearFile, + } = useFileHandler(); + + const { + messages, + memories, + thinking, + sendMessage, + } = useChat({ + user: selectedUser, + mem0ApiKey: selectedMem0Key, + openaiApiKey: selectedOpenAIKey, + provider: selectedProvider, + }); + + const handleSend = async (content: string) => { + if (file) { + await sendMessage(content, { + type: file.type, + data: fileData!, + }); + clearFile(); + } else { + await sendMessage(content); + } + }; + + const setFile = async (newFile: File | null) => { + if (newFile) { + await handleFile(newFile); + } else { + clearFile(); + } + }; + + return ( + + {props.children} + + ); +}; + +export default GlobalContext; +export { GlobalState }; \ No newline at end of file diff --git a/examples/multimodal-demo/src/hooks/useAuth.ts b/examples/multimodal-demo/src/hooks/useAuth.ts new file mode 100644 index 00000000..5687442c --- /dev/null +++ b/examples/multimodal-demo/src/hooks/useAuth.ts @@ -0,0 +1,73 @@ +import { useState, useEffect } from 'react'; +import { Provider } from '@/constants/messages'; + +interface UseAuthReturn { + mem0ApiKey: string; + openaiApiKey: string; + provider: Provider; + user: string; + setAuth: (mem0: string, openai: string, provider: Provider) => void; + setUser: (user: string) => void; + clearAuth: () => void; + clearUser: () => void; +} + +export const useAuth = (): UseAuthReturn => { + const [mem0ApiKey, setMem0ApiKey] = useState(''); + const [openaiApiKey, setOpenaiApiKey] = useState(''); + const [provider, setProvider] = useState('openai'); + const [user, setUser] = useState(''); + + useEffect(() => { + const mem0 = localStorage.getItem('mem0ApiKey'); + const openai = localStorage.getItem('openaiApiKey'); + const savedProvider = localStorage.getItem('provider') as Provider; + const savedUser = localStorage.getItem('user'); + + if (mem0 && openai && savedProvider) { + setAuth(mem0, openai, savedProvider); + } + if (savedUser) { + setUser(savedUser); + } + }, []); + + const setAuth = (mem0: string, openai: string, provider: Provider) => { + setMem0ApiKey(mem0); + setOpenaiApiKey(openai); + setProvider(provider); + localStorage.setItem('mem0ApiKey', mem0); + localStorage.setItem('openaiApiKey', openai); + localStorage.setItem('provider', provider); + }; + + const clearAuth = () => { + localStorage.removeItem('mem0ApiKey'); + localStorage.removeItem('openaiApiKey'); + localStorage.removeItem('provider'); + setMem0ApiKey(''); + setOpenaiApiKey(''); + setProvider('openai'); + }; + + const updateUser = (user: string) => { + setUser(user); + localStorage.setItem('user', user); + }; + + const clearUser = () => { + localStorage.removeItem('user'); + setUser(''); + }; + + return { + mem0ApiKey, + openaiApiKey, + provider, + user, + setAuth, + setUser: updateUser, + clearAuth, + clearUser, + }; +}; \ No newline at end of file diff --git a/examples/multimodal-demo/src/hooks/useChat.ts b/examples/multimodal-demo/src/hooks/useChat.ts new file mode 100644 index 00000000..b54826a9 --- /dev/null +++ b/examples/multimodal-demo/src/hooks/useChat.ts @@ -0,0 +1,220 @@ +import { useState } from 'react'; +import { MemoryClient } from 'saket-test'; +import { OpenAI } from 'openai'; +import { Message, Memory } from '@/types'; +import { WELCOME_MESSAGE, INVALID_CONFIG_MESSAGE, ERROR_MESSAGE, AI_MODELS, Provider } from '@/constants/messages'; + +interface UseChatProps { + user: string; + mem0ApiKey: string; + openaiApiKey: string; + provider: Provider; +} + +interface UseChatReturn { + messages: Message[]; + memories: Memory[]; + thinking: boolean; + sendMessage: (content: string, fileData?: { type: string; data: string | Buffer }) => Promise; +} + +type MessageContent = string | { + type: 'image_url'; + image_url: { + url: string; + }; +}; + +interface PromptMessage { + role: string; + content: MessageContent; +} + +export const useChat = ({ user, mem0ApiKey, openaiApiKey, provider }: UseChatProps): UseChatReturn => { + const [messages, setMessages] = useState([WELCOME_MESSAGE]); + const [memories, setMemories] = useState([]); + const [thinking, setThinking] = useState(false); + + const openai = new OpenAI({ apiKey: openaiApiKey}); + const memoryClient = new MemoryClient({ apiKey: mem0ApiKey }); + + const updateMemories = async (messages: PromptMessage[]) => { + console.log(messages); + try { + await memoryClient.add(messages, { + user_id: user, + output_format: "v1.1", + }); + + const response = await memoryClient.getAll({ + user_id: user, + page: 1, + page_size: 50, + }); + + const newMemories = response.results.map((memory: any) => ({ + id: memory.id, + content: memory.memory, + timestamp: memory.updated_at, + tags: memory.categories || [], + })); + setMemories(newMemories); + } catch (error) { + console.error('Error in updateMemories:', error); + } + }; + + const formatMessagesForPrompt = (messages: Message[]): PromptMessage[] => { + return messages.map((message) => { + if (message.image) { + return { + role: message.sender, + content: { + type: 'image_url', + image_url: { + url: message.image + } + }, + }; + } + + return { + role: message.sender, + content: message.content, + }; + }); + }; + + const sendMessage = async (content: string, fileData?: { type: string; data: string | Buffer }) => { + if (!content.trim() && !fileData) return; + + if (!user) { + const newMessage: Message = { + id: Date.now().toString(), + content, + sender: 'user', + timestamp: new Date().toLocaleTimeString(), + }; + setMessages((prev) => [...prev, newMessage, INVALID_CONFIG_MESSAGE]); + return; + } + + const userMessage: Message = { + id: Date.now().toString(), + content, + sender: 'user', + timestamp: new Date().toLocaleTimeString(), + ...(fileData?.type.startsWith('image/') && { image: fileData.data.toString() }), + }; + + setMessages((prev) => [...prev, userMessage]); + setThinking(true); + + // Get all messages for memory update + const allMessagesForMemory = formatMessagesForPrompt([...messages, userMessage]); + await updateMemories(allMessagesForMemory); + + try { + // Get only the last assistant message (if exists) and the current user message + const lastAssistantMessage = messages.filter(msg => msg.sender === 'assistant').slice(-1)[0]; + let messagesForLLM = lastAssistantMessage + ? [ + formatMessagesForPrompt([lastAssistantMessage])[0], + formatMessagesForPrompt([userMessage])[0] + ] + : [formatMessagesForPrompt([userMessage])[0]]; + + // Check if any message has image content + const hasImage = messagesForLLM.some(msg => { + if (typeof msg.content === 'object' && msg.content !== null) { + const content = msg.content as any; + return content.type === 'image_url'; + } + return false; + }); + + // For image messages, only use the text content + if (hasImage) { + messagesForLLM = [{ + role: 'user', + content: userMessage.content + }]; + } + + // Fetch relevant memories if there's an image + let relevantMemories = ''; + if (hasImage) { + try { + const searchResponse = await memoryClient.getAll({ + user_id: user, + page: 1, + page_size: 10, + }); + + relevantMemories = searchResponse.results + .map((memory: any) => `Previous context: ${memory.memory}`) + .join('\n'); + } catch (error) { + console.error('Error fetching memories:', error); + } + } + + // Add a system message with memories context if there are memories and image + if (relevantMemories.length > 0 && hasImage) { + messagesForLLM = [ + { + role: 'system', + content: `Here are some relevant details about the user:\n${relevantMemories}\n\nPlease use this context when responding to the user's message.` + }, + ...messagesForLLM + ]; + } + + console.log('Messages for LLM:', messagesForLLM); + const completion = await openai.chat.completions.create({ + model: "gpt-4", + messages: messagesForLLM.map(msg => ({ + role: msg.role, + content: msg.content + })), + stream: true, + }); + + const assistantMessageId = Date.now() + 1; + const assistantMessage: Message = { + id: assistantMessageId.toString(), + content: '', + sender: 'assistant', + timestamp: new Date().toLocaleTimeString(), + }; + + setMessages((prev) => [...prev, assistantMessage]); + + for await (const chunk of completion) { + const textPart = chunk.choices[0]?.delta?.content || ''; + assistantMessage.content += textPart; + setThinking(false); + + setMessages((prev) => + prev.map((msg) => + msg.id === assistantMessageId.toString() + ? { ...msg, content: assistantMessage.content } + : msg + ) + ); + } + } catch (error) { + console.error('Error in sendMessage:', error); + setMessages((prev) => [...prev, ERROR_MESSAGE]); + } finally { + setThinking(false); + } + }; + + return { + messages, + memories, + thinking, + sendMessage, + }; +}; \ No newline at end of file diff --git a/examples/multimodal-demo/src/hooks/useFileHandler.ts b/examples/multimodal-demo/src/hooks/useFileHandler.ts new file mode 100644 index 00000000..3353a8cf --- /dev/null +++ b/examples/multimodal-demo/src/hooks/useFileHandler.ts @@ -0,0 +1,45 @@ +import { useState } from 'react'; +import { FileInfo } from '@/types'; +import { convertToBase64, getFileBuffer } from '@/utils/fileUtils'; + +interface UseFileHandlerReturn { + selectedFile: FileInfo | null; + file: File | null; + fileData: string | Buffer | null; + setSelectedFile: (file: FileInfo | null) => void; + handleFile: (file: File) => Promise; + clearFile: () => void; +} + +export const useFileHandler = (): UseFileHandlerReturn => { + const [selectedFile, setSelectedFile] = useState(null); + const [file, setFile] = useState(null); + const [fileData, setFileData] = useState(null); + + const handleFile = async (file: File) => { + setFile(file); + + if (file.type.startsWith('image/')) { + const base64Data = await convertToBase64(file); + setFileData(base64Data); + } else if (file.type.startsWith('audio/')) { + const bufferData = await getFileBuffer(file); + setFileData(bufferData); + } + }; + + const clearFile = () => { + setSelectedFile(null); + setFile(null); + setFileData(null); + }; + + return { + selectedFile, + file, + fileData, + setSelectedFile, + handleFile, + clearFile, + }; +}; \ No newline at end of file diff --git a/examples/multimodal-demo/src/index.css b/examples/multimodal-demo/src/index.css new file mode 100644 index 00000000..405a75d5 --- /dev/null +++ b/examples/multimodal-demo/src/index.css @@ -0,0 +1,97 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; +@layer base { + :root { + --background: 0 0% 100%; + --foreground: 240 10% 3.9%; + --card: 0 0% 100%; + --card-foreground: 240 10% 3.9%; + --popover: 0 0% 100%; + --popover-foreground: 240 10% 3.9%; + --primary: 240 5.9% 10%; + --primary-foreground: 0 0% 98%; + --secondary: 240 4.8% 95.9%; + --secondary-foreground: 240 5.9% 10%; + --muted: 240 4.8% 95.9%; + --muted-foreground: 240 3.8% 46.1%; + --accent: 240 4.8% 95.9%; + --accent-foreground: 240 5.9% 10%; + --destructive: 0 84.2% 60.2%; + --destructive-foreground: 0 0% 98%; + --border: 240 5.9% 90%; + --input: 240 5.9% 90%; + --ring: 240 10% 3.9%; + --chart-1: 12 76% 61%; + --chart-2: 173 58% 39%; + --chart-3: 197 37% 24%; + --chart-4: 43 74% 66%; + --chart-5: 27 87% 67%; + --radius: 0.5rem + } + .dark { + --background: 240 10% 3.9%; + --foreground: 0 0% 98%; + --card: 240 10% 3.9%; + --card-foreground: 0 0% 98%; + --popover: 240 10% 3.9%; + --popover-foreground: 0 0% 98%; + --primary: 0 0% 98%; + --primary-foreground: 240 5.9% 10%; + --secondary: 240 3.7% 15.9%; + --secondary-foreground: 0 0% 98%; + --muted: 240 3.7% 15.9%; + --muted-foreground: 240 5% 64.9%; + --accent: 240 3.7% 15.9%; + --accent-foreground: 0 0% 98%; + --destructive: 0 62.8% 30.6%; + --destructive-foreground: 0 0% 98%; + --border: 240 3.7% 15.9%; + --input: 240 3.7% 15.9%; + --ring: 240 4.9% 83.9%; + --chart-1: 220 70% 50%; + --chart-2: 160 60% 45%; + --chart-3: 30 80% 55%; + --chart-4: 280 65% 60%; + --chart-5: 340 75% 55% + } +} +@layer base { + * { + @apply border-border; + } + body { + @apply bg-background text-foreground; + } +} + +.loader { + display: flex; + align-items: flex-end; + gap: 5px; +} + +.ball { + width: 6px; + height: 6px; + background-color: #4e4e4e; + border-radius: 50%; + animation: bounce 0.6s infinite alternate; +} + +.ball:nth-child(2) { + animation-delay: 0.2s; +} + +.ball:nth-child(3) { + animation-delay: 0.4s; +} + +@keyframes bounce { + from { + transform: translateY(0); + } + to { + transform: translateY(-4px); + } +} diff --git a/examples/multimodal-demo/src/libs/utils.ts b/examples/multimodal-demo/src/libs/utils.ts new file mode 100644 index 00000000..bd0c391d --- /dev/null +++ b/examples/multimodal-demo/src/libs/utils.ts @@ -0,0 +1,6 @@ +import { clsx, type ClassValue } from "clsx" +import { twMerge } from "tailwind-merge" + +export function cn(...inputs: ClassValue[]) { + return twMerge(clsx(inputs)) +} diff --git a/examples/multimodal-demo/src/main.tsx b/examples/multimodal-demo/src/main.tsx new file mode 100644 index 00000000..bef5202a --- /dev/null +++ b/examples/multimodal-demo/src/main.tsx @@ -0,0 +1,10 @@ +import { StrictMode } from 'react' +import { createRoot } from 'react-dom/client' +import './index.css' +import App from './App.tsx' + +createRoot(document.getElementById('root')!).render( + + + , +) diff --git a/examples/multimodal-demo/src/page.tsx b/examples/multimodal-demo/src/page.tsx new file mode 100644 index 00000000..1f99e856 --- /dev/null +++ b/examples/multimodal-demo/src/page.tsx @@ -0,0 +1,14 @@ +"use client"; +import { GlobalState } from "./contexts/GlobalContext"; +import Component from "./pages/home"; + + +export default function Home() { + return ( +
+ + + +
+ ); +} diff --git a/examples/multimodal-demo/src/pages/home.tsx b/examples/multimodal-demo/src/pages/home.tsx new file mode 100644 index 00000000..f72b175e --- /dev/null +++ b/examples/multimodal-demo/src/pages/home.tsx @@ -0,0 +1,41 @@ +import { useState } from "react"; +import ApiSettingsPopup from "../components/api-settings-popup"; +import Memories from "../components/memories"; +import Header from "../components/header"; +import Messages from "../components/messages"; +import InputArea from "../components/input-area"; +import ChevronToggle from "../components/chevron-toggle"; + + +export default function Home() { + const [isMemoriesExpanded, setIsMemoriesExpanded] = useState(true); + const [isSettingsOpen, setIsSettingsOpen] = useState(false); + + return ( + <> + +
+ {/* Main Chat Area */} +
+ {/* Header */} +
+ + {/* Messages */} + + + {/* Input Area */} + +
+ + {/* Chevron Toggle */} + + + {/* Memories Sidebar */} + +
+ + ); +} diff --git a/examples/multimodal-demo/src/types.ts b/examples/multimodal-demo/src/types.ts new file mode 100644 index 00000000..770bc23f --- /dev/null +++ b/examples/multimodal-demo/src/types.ts @@ -0,0 +1,22 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +export interface Memory { + id: string; + content: string; + timestamp: string; + tags: string[]; +} + +export interface Message { + id: string; + content: string; + sender: "user" | "assistant"; + timestamp: string; + image?: string; + audio?: any; +} + +export interface FileInfo { + name: string; + type: string; + size: number; +} \ No newline at end of file diff --git a/examples/multimodal-demo/src/utils/fileUtils.ts b/examples/multimodal-demo/src/utils/fileUtils.ts new file mode 100644 index 00000000..cd86f807 --- /dev/null +++ b/examples/multimodal-demo/src/utils/fileUtils.ts @@ -0,0 +1,16 @@ +import { Buffer } from 'buffer'; + +export const convertToBase64 = (file: File): Promise => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(file); + reader.onload = () => resolve(reader.result as string); + reader.onerror = error => reject(error); + }); +}; + +export const getFileBuffer = async (file: File): Promise => { + const response = await fetch(URL.createObjectURL(file)); + const arrayBuffer = await response.arrayBuffer(); + return Buffer.from(arrayBuffer); +}; \ No newline at end of file diff --git a/examples/multimodal-demo/src/vite-env.d.ts b/examples/multimodal-demo/src/vite-env.d.ts new file mode 100644 index 00000000..11f02fe2 --- /dev/null +++ b/examples/multimodal-demo/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/examples/multimodal-demo/tailwind.config.js b/examples/multimodal-demo/tailwind.config.js new file mode 100644 index 00000000..15012851 --- /dev/null +++ b/examples/multimodal-demo/tailwind.config.js @@ -0,0 +1,62 @@ +// tailwind.config.js +/* eslint-env node */ + +/** @type {import('tailwindcss').Config} */ +import tailwindcssAnimate from 'tailwindcss-animate'; + +export default { + darkMode: ["class"], + content: ["./index.html", "./src/**/*.{ts,tsx,js,jsx}"], + theme: { + extend: { + borderRadius: { + lg: 'var(--radius)', + md: 'calc(var(--radius) - 2px)', + sm: 'calc(var(--radius) - 4px)', + }, + colors: { + background: 'hsl(var(--background))', + foreground: 'hsl(var(--foreground))', + card: { + DEFAULT: 'hsl(var(--card))', + foreground: 'hsl(var(--card-foreground))', + }, + popover: { + DEFAULT: 'hsl(var(--popover))', + foreground: 'hsl(var(--popover-foreground))', + }, + primary: { + DEFAULT: 'hsl(var(--primary))', + foreground: 'hsl(var(--primary-foreground))', + }, + secondary: { + DEFAULT: 'hsl(var(--secondary))', + foreground: 'hsl(var(--secondary-foreground))', + }, + muted: { + DEFAULT: 'hsl(var(--muted))', + foreground: 'hsl(var(--muted-foreground))', + }, + accent: { + DEFAULT: 'hsl(var(--accent))', + foreground: 'hsl(var(--accent-foreground))', + }, + destructive: { + DEFAULT: 'hsl(var(--destructive))', + foreground: 'hsl(var(--destructive-foreground))', + }, + border: 'hsl(var(--border))', + input: 'hsl(var(--input))', + ring: 'hsl(var(--ring))', + chart: { + '1': 'hsl(var(--chart-1))', + '2': 'hsl(var(--chart-2))', + '3': 'hsl(var(--chart-3))', + '4': 'hsl(var(--chart-4))', + '5': 'hsl(var(--chart-5))', + }, + }, + }, + }, + plugins: [tailwindcssAnimate], +}; diff --git a/examples/multimodal-demo/tsconfig.app.json b/examples/multimodal-demo/tsconfig.app.json new file mode 100644 index 00000000..6d0c89af --- /dev/null +++ b/examples/multimodal-demo/tsconfig.app.json @@ -0,0 +1,32 @@ +{ + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", + "target": "ES2020", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "baseUrl": ".", + "paths": { + "@/*": [ + "./src/*" + ] + }, + + /* Bundler mode */ + "moduleResolution": "Bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + + /* Linting */ + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true + }, + "include": ["src"] +} diff --git a/examples/multimodal-demo/tsconfig.json b/examples/multimodal-demo/tsconfig.json new file mode 100644 index 00000000..fec8c8e5 --- /dev/null +++ b/examples/multimodal-demo/tsconfig.json @@ -0,0 +1,13 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.app.json" }, + { "path": "./tsconfig.node.json" } + ], + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@/*": ["./src/*"] + } + } +} diff --git a/examples/multimodal-demo/tsconfig.node.json b/examples/multimodal-demo/tsconfig.node.json new file mode 100644 index 00000000..abcd7f0d --- /dev/null +++ b/examples/multimodal-demo/tsconfig.node.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", + "target": "ES2022", + "lib": ["ES2023"], + "module": "ESNext", + "skipLibCheck": true, + + /* Bundler mode */ + "moduleResolution": "Bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + + /* Linting */ + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true + }, + "include": ["vite.config.ts"] +} diff --git a/examples/multimodal-demo/useChat.ts b/examples/multimodal-demo/useChat.ts new file mode 100644 index 00000000..4f3f37c1 --- /dev/null +++ b/examples/multimodal-demo/useChat.ts @@ -0,0 +1,223 @@ +import { useState } from 'react'; +import { MemoryClient, Memory as Mem0Memory } from 'mem0ai'; +import { OpenAI } from 'openai'; +import { Message, Memory } from '@/types'; +import { WELCOME_MESSAGE, INVALID_CONFIG_MESSAGE, ERROR_MESSAGE, Provider } from '@/constants/messages'; + +interface UseChatProps { + user: string; + mem0ApiKey: string; + openaiApiKey: string; + provider: Provider; +} + +interface UseChatReturn { + messages: Message[]; + memories: Memory[]; + thinking: boolean; + sendMessage: (content: string, fileData?: { type: string; data: string | Buffer }) => Promise; +} + +type MessageContent = string | { + type: 'image_url'; + image_url: { + url: string; + }; +}; + +interface PromptMessage { + role: string; + content: MessageContent; +} + +export const useChat = ({ user, mem0ApiKey, openaiApiKey }: UseChatProps): UseChatReturn => { + const [messages, setMessages] = useState([WELCOME_MESSAGE]); + const [memories, setMemories] = useState(); + const [thinking, setThinking] = useState(false); + + const openai = new OpenAI({ apiKey: openaiApiKey, dangerouslyAllowBrowser: true}); + + const updateMemories = async (messages: PromptMessage[]) => { + const memoryClient = new MemoryClient({ apiKey: mem0ApiKey || '' }); + try { + await memoryClient.add(messages, { + user_id: user, + }); + + const response = await memoryClient.getAll({ + user_id: user, + }); + + const newMemories = response.map((memory: Mem0Memory) => ({ + id: memory.id || '', + content: memory.memory || '', + timestamp: String(memory.updated_at) || '', + tags: memory.categories || [], + })); + setMemories(newMemories); + } catch (error) { + console.error('Error in updateMemories:', error); + } + }; + + const formatMessagesForPrompt = (messages: Message[]): PromptMessage[] => { + return messages.map((message) => { + if (message.image) { + return { + role: message.sender, + content: { + type: 'image_url', + image_url: { + url: message.image + } + }, + }; + } + + return { + role: message.sender, + content: message.content, + }; + }); + }; + + const sendMessage = async (content: string, fileData?: { type: string; data: string | Buffer }) => { + if (!content.trim() && !fileData) return; + + const memoryClient = new MemoryClient({ apiKey: mem0ApiKey || '' }); + + if (!user) { + const newMessage: Message = { + id: Date.now().toString(), + content, + sender: 'user', + timestamp: new Date().toLocaleTimeString(), + }; + setMessages((prev) => [...prev, newMessage, INVALID_CONFIG_MESSAGE]); + return; + } + + const userMessage: Message = { + id: Date.now().toString(), + content, + sender: 'user', + timestamp: new Date().toLocaleTimeString(), + ...(fileData?.type.startsWith('image/') && { image: fileData.data.toString() }), + }; + + setMessages((prev) => [...prev, userMessage]); + setThinking(true); + + // Get all messages for memory update + const allMessagesForMemory = formatMessagesForPrompt([...messages, userMessage]); + await updateMemories(allMessagesForMemory); + + try { + // Get only the last assistant message (if exists) and the current user message + const lastAssistantMessage = messages.filter(msg => msg.sender === 'assistant').slice(-1)[0]; + let messagesForLLM = lastAssistantMessage + ? [ + formatMessagesForPrompt([lastAssistantMessage])[0], + formatMessagesForPrompt([userMessage])[0] + ] + : [formatMessagesForPrompt([userMessage])[0]]; + + // Check if any message has image content + const hasImage = messagesForLLM.some(msg => { + if (typeof msg.content === 'object' && msg.content !== null) { + const content = msg.content as MessageContent; + return typeof content === 'object' && content !== null && 'type' in content && content.type === 'image_url'; + } + return false; + }); + + // For image messages, only use the text content + if (hasImage) { + messagesForLLM = [ + ...messagesForLLM, + { + role: 'user', + content: userMessage.content + } + ]; + } + + // Fetch relevant memories if there's an image + let relevantMemories = ''; + try { + const searchResponse = await memoryClient.getAll({ + user_id: user + }); + + relevantMemories = searchResponse + .map((memory: Mem0Memory) => `Previous context: ${memory.memory}`) + .join('\n'); + } catch (error) { + console.error('Error fetching memories:', error); + } + + // Add a system message with memories context if there are memories and image + if (relevantMemories.length > 0 && hasImage) { + messagesForLLM = [ + { + role: 'system', + content: `Here are some relevant details about the user:\n${relevantMemories}\n\nPlease use this context when responding to the user's message.` + }, + ...messagesForLLM + ]; + } + + const generateRandomId = () => { + return Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15); + } + + const completion = await openai.chat.completions.create({ + model: "gpt-4o-mini", + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + messages: messagesForLLM.map(msg => ({ + role: msg.role === 'user' ? 'user' : 'assistant', + content: typeof msg.content === 'object' && msg.content !== null ? [msg.content] : msg.content, + name: generateRandomId(), + })), + stream: true, + }); + + const assistantMessageId = Date.now() + 1; + const assistantMessage: Message = { + id: assistantMessageId.toString(), + content: '', + sender: 'assistant', + timestamp: new Date().toLocaleTimeString(), + }; + + setMessages((prev) => [...prev, assistantMessage]); + + for await (const chunk of completion) { + const textPart = chunk.choices[0]?.delta?.content || ''; + assistantMessage.content += textPart; + setThinking(false); + + setMessages((prev) => + prev.map((msg) => + msg.id === assistantMessageId.toString() + ? { ...msg, content: assistantMessage.content } + : msg + ) + ); + } + } catch (error) { + console.error('Error in sendMessage:', error); + setMessages((prev) => [...prev, ERROR_MESSAGE]); + } finally { + setThinking(false); + } + }; + + return { + messages, + memories: memories || [], + thinking, + sendMessage, + }; +}; \ No newline at end of file diff --git a/examples/multimodal-demo/vite.config.ts b/examples/multimodal-demo/vite.config.ts new file mode 100644 index 00000000..a761a870 --- /dev/null +++ b/examples/multimodal-demo/vite.config.ts @@ -0,0 +1,13 @@ +import path from "path" +import react from "@vitejs/plugin-react" +import { defineConfig } from "vite" + +export default defineConfig({ + plugins: [react()], + resolve: { + alias: { + "@": path.resolve(__dirname, "./src"), + buffer: 'buffer' + }, + }, +})