From 4a8c50f8867ab90f92130d1dc25209231686f5a2 Mon Sep 17 00:00:00 2001 From: Deshraj Yadav Date: Fri, 13 Oct 2023 15:38:15 -0700 Subject: [PATCH] [docs]: Revamp embedchain docs (#799) --- .github/workflows/cd.yml | 4 +- configs/anthropic.yaml | 8 + {embedchain/yaml => configs}/chroma.yaml | 0 configs/cohere.yaml | 7 + configs/full-stack.yaml | 35 +++ configs/gpt4all.yaml | 13 + configs/huggingface.yaml | 8 + configs/jina.yaml | 7 + configs/llama2.yaml | 8 + {embedchain/yaml => configs}/opensearch.yaml | 0 {embedchain/yaml => configs}/opensource.yaml | 0 configs/vertexai.yaml | 6 + docs/Makefile | 10 + docs/_snippets/get-help.mdx | 11 + docs/_snippets/missing-data-source-tip.mdx | 18 ++ docs/_snippets/missing-llm-tip.mdx | 18 ++ docs/_snippets/missing-vector-db-tip.mdx | 18 ++ docs/advanced/app_types.mdx | 215 -------------- docs/advanced/configuration.mdx | 143 ++++----- docs/advanced/interface_types.mdx | 75 ----- docs/advanced/query_configuration.mdx | 15 +- docs/components/embedding-models.mdx | 135 +++++++++ docs/components/llms.mdx | 280 ++++++++++++++++++ docs/components/vector-databases.mdx | 142 +++++++++ docs/data-sources/csv.mdx | 17 +- docs/data-sources/data-type-handling.mdx | 12 +- docs/data-sources/docs-site.mdx | 11 +- docs/data-sources/docx.mdx | 12 +- docs/data-sources/how-to-add-data.mdx | 24 -- docs/data-sources/mdx.mdx | 14 +- docs/data-sources/notion.mdx | 19 +- docs/data-sources/overview.mdx | 24 ++ docs/data-sources/pdf-file.mdx | 13 +- docs/data-sources/qna.mdx | 8 +- docs/data-sources/request-new-format.mdx | 4 - docs/data-sources/sitemap.mdx | 8 +- docs/data-sources/text.mdx | 8 +- docs/data-sources/web-page.mdx | 10 +- docs/data-sources/xml.mdx | 8 +- docs/data-sources/youtube-video.mdx | 7 +- docs/get-start/faq.mdx | 16 - docs/get-start/introduction.mdx | 60 ---- docs/get-start/quickstart.mdx | 35 --- docs/get-started/faq.mdx | 68 +++++ docs/get-started/introduction.mdx | 58 ++++ docs/get-started/quickstart.mdx | 52 ++++ docs/mint.json | 46 +-- embedchain/apps/app.py | 2 +- embedchain/apps/open_source_app.py | 4 +- embedchain/apps/person_app.py | 4 +- embedchain/bots/base.py | 6 +- embedchain/config/__init__.py | 3 +- .../llm/{base_llm_config.py => base.py} | 4 +- embedchain/embedchain.py | 4 +- embedchain/embedder/vertexai.py | 2 +- embedchain/factory.py | 10 +- embedchain/llm/base.py | 10 +- embedchain/llm/gpt4all.py | 4 +- .../{hugging_face_hub.py => huggingface.py} | 12 +- embedchain/llm/vertex_ai.py | 4 +- pyproject.toml | 13 +- tests/apps/test_apps.py | 4 +- tests/llm/test_chat.py | 4 +- tests/llm/test_generate_prompt.py | 4 +- ...ugging_face_hub.py => test_huggingface.py} | 28 +- tests/llm/test_query.py | 6 +- tests/llm/test_vertex_ai.py | 6 +- tests/test_factory.py | 4 +- 68 files changed, 1175 insertions(+), 673 deletions(-) create mode 100644 configs/anthropic.yaml rename {embedchain/yaml => configs}/chroma.yaml (100%) create mode 100644 configs/cohere.yaml create mode 100644 configs/full-stack.yaml create mode 100644 configs/gpt4all.yaml create mode 100644 configs/huggingface.yaml create mode 100644 configs/jina.yaml create mode 100644 configs/llama2.yaml rename {embedchain/yaml => configs}/opensearch.yaml (100%) rename {embedchain/yaml => configs}/opensource.yaml (100%) create mode 100644 configs/vertexai.yaml create mode 100644 docs/Makefile create mode 100644 docs/_snippets/get-help.mdx create mode 100644 docs/_snippets/missing-data-source-tip.mdx create mode 100644 docs/_snippets/missing-llm-tip.mdx create mode 100644 docs/_snippets/missing-vector-db-tip.mdx delete mode 100644 docs/advanced/app_types.mdx delete mode 100644 docs/advanced/interface_types.mdx create mode 100644 docs/components/embedding-models.mdx create mode 100644 docs/components/llms.mdx create mode 100644 docs/components/vector-databases.mdx delete mode 100644 docs/data-sources/how-to-add-data.mdx create mode 100644 docs/data-sources/overview.mdx delete mode 100644 docs/data-sources/request-new-format.mdx delete mode 100644 docs/get-start/faq.mdx delete mode 100644 docs/get-start/introduction.mdx delete mode 100644 docs/get-start/quickstart.mdx create mode 100644 docs/get-started/faq.mdx create mode 100644 docs/get-started/introduction.mdx create mode 100644 docs/get-started/quickstart.mdx rename embedchain/config/llm/{base_llm_config.py => base.py} (98%) rename embedchain/llm/{hugging_face_hub.py => huggingface.py} (76%) rename tests/llm/{test_hugging_face_hub.py => test_huggingface.py} (68%) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index c357e514..dacbfef6 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -23,10 +23,10 @@ jobs: run: | curl -sSL https://install.python-poetry.org | python3 - echo "$HOME/.local/bin" >> $GITHUB_PATH - + - name: Install dependencies run: poetry install - + - name: Build a binary wheel and a source tarball run: poetry build diff --git a/configs/anthropic.yaml b/configs/anthropic.yaml new file mode 100644 index 00000000..e3b50ecf --- /dev/null +++ b/configs/anthropic.yaml @@ -0,0 +1,8 @@ +llm: + provider: anthropic + model: 'claude-instant-1' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 + stream: false diff --git a/embedchain/yaml/chroma.yaml b/configs/chroma.yaml similarity index 100% rename from embedchain/yaml/chroma.yaml rename to configs/chroma.yaml diff --git a/configs/cohere.yaml b/configs/cohere.yaml new file mode 100644 index 00000000..8a799506 --- /dev/null +++ b/configs/cohere.yaml @@ -0,0 +1,7 @@ +llm: + provider: cohere + model: large + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 diff --git a/configs/full-stack.yaml b/configs/full-stack.yaml new file mode 100644 index 00000000..13b661f8 --- /dev/null +++ b/configs/full-stack.yaml @@ -0,0 +1,35 @@ +app: + config: + id: 'full-stack-app' + +llm: + provider: openai + model: 'gpt-3.5-turbo' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 + stream: false + template: | + Use the following pieces of context to answer the query at the end. + If you don't know the answer, just say that you don't know, don't try to make up an answer. + + $context + + Query: $query + + Helpful Answer: + system_prompt: | + Act as William Shakespeare. Answer the following questions in the style of William Shakespeare. + +vectordb: + provider: chroma + config: + collection_name: 'full-stack-app' + dir: db + allow_reset: true + +embedder: + provider: openai + config: + model: 'text-embedding-ada-002' diff --git a/configs/gpt4all.yaml b/configs/gpt4all.yaml new file mode 100644 index 00000000..e7bba542 --- /dev/null +++ b/configs/gpt4all.yaml @@ -0,0 +1,13 @@ +llm: + provider: gpt4all + model: 'orca-mini-3b.ggmlv3.q4_0.bin' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 + stream: false + +embedder: + provider: gpt4all + config: + model: 'all-MiniLM-L6-v2' diff --git a/configs/huggingface.yaml b/configs/huggingface.yaml new file mode 100644 index 00000000..f9347f68 --- /dev/null +++ b/configs/huggingface.yaml @@ -0,0 +1,8 @@ +llm: + provider: huggingface + model: 'google/flan-t5-xxl' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 0.5 + stream: false diff --git a/configs/jina.yaml b/configs/jina.yaml new file mode 100644 index 00000000..11627059 --- /dev/null +++ b/configs/jina.yaml @@ -0,0 +1,7 @@ +llm: + provider: jina + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 + stream: false diff --git a/configs/llama2.yaml b/configs/llama2.yaml new file mode 100644 index 00000000..fb9f5002 --- /dev/null +++ b/configs/llama2.yaml @@ -0,0 +1,8 @@ +llm: + provider: llama2 + model: 'a16z-infra/llama13b-v2-chat:df7690f1994d94e96ad9d568eac121aecf50684a0b0963b25a41cc40061269e5' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 0.5 + stream: false diff --git a/embedchain/yaml/opensearch.yaml b/configs/opensearch.yaml similarity index 100% rename from embedchain/yaml/opensearch.yaml rename to configs/opensearch.yaml diff --git a/embedchain/yaml/opensource.yaml b/configs/opensource.yaml similarity index 100% rename from embedchain/yaml/opensource.yaml rename to configs/opensource.yaml diff --git a/configs/vertexai.yaml b/configs/vertexai.yaml new file mode 100644 index 00000000..83e7a683 --- /dev/null +++ b/configs/vertexai.yaml @@ -0,0 +1,6 @@ +llm: + provider: vertexai + model: 'chat-bison' + config: + temperature: 0.5 + top_p: 0.5 diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..0db640d0 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,10 @@ +install: + npm i -g mintlify + +run_local: + mintlify dev + +troubleshoot: + mintlify install + +.PHONY: install run_local troubleshoot diff --git a/docs/_snippets/get-help.mdx b/docs/_snippets/get-help.mdx new file mode 100644 index 00000000..c6193f0d --- /dev/null +++ b/docs/_snippets/get-help.mdx @@ -0,0 +1,11 @@ + + + Join our slack community + + + Join our discord community + + + Schedule a call with Embedchain founder + + diff --git a/docs/_snippets/missing-data-source-tip.mdx b/docs/_snippets/missing-data-source-tip.mdx new file mode 100644 index 00000000..fbb768a3 --- /dev/null +++ b/docs/_snippets/missing-data-source-tip.mdx @@ -0,0 +1,18 @@ + +If you can't find the specific data source, please feel free to request through one of the following channels and help us prioritize. + + + + Let us know on our slack community + + + Let us know on discord community + + + Open an issue on our GitHub + + + Schedule a call with Embedchain founder + + + diff --git a/docs/_snippets/missing-llm-tip.mdx b/docs/_snippets/missing-llm-tip.mdx new file mode 100644 index 00000000..e9fc2561 --- /dev/null +++ b/docs/_snippets/missing-llm-tip.mdx @@ -0,0 +1,18 @@ + +If you can't find the specific LLM you need, no need to fret. We're continuously expanding our support for additional LLMs, and you can help us prioritize by opening an issue on our GitHub or simply reaching out to us on our Slack or Discord community. + + + + Let us know on our slack community + + + Let us know on discord community + + + Open an issue on our GitHub + + + Schedule a call with Embedchain founder + + + diff --git a/docs/_snippets/missing-vector-db-tip.mdx b/docs/_snippets/missing-vector-db-tip.mdx new file mode 100644 index 00000000..8b9dd222 --- /dev/null +++ b/docs/_snippets/missing-vector-db-tip.mdx @@ -0,0 +1,18 @@ + +If you can't find the specific vector database, please feel free to request through one of the following channels and help us prioritize. + + + + Let us know on our slack community + + + Let us know on discord community + + + Open an issue on our GitHub + + + Schedule a call with Embedchain founder + + + diff --git a/docs/advanced/app_types.mdx b/docs/advanced/app_types.mdx deleted file mode 100644 index d58a9dca..00000000 --- a/docs/advanced/app_types.mdx +++ /dev/null @@ -1,215 +0,0 @@ ---- -title: 'πŸ“± App types' ---- - -## App Types - -Embedchain supports a variety of LLMs, embedding functions/models and vector databases. - -Our app gives you full control over which components you want to use, you can mix and match them to your hearts content. - - -Out of the box, if you just use `app = App()`, Embedchain uses what we believe to be the best configuration available. This might include paid/proprietary components. Currently, this is - -* LLM: OpenAi (gpt-3.5-turbo) -* Embedder: OpenAi (text-embedding-ada-002) -* Database: ChromaDB - - -### LLM - -#### Choosing an LLM - -The following LLM providers are supported by Embedchain: -- OPENAI -- ANTHPROPIC -- VERTEX_AI -- GPT4ALL -- AZURE_OPENAI -- LLAMA2 -- JINA -- COHERE - -You can choose one by importing it from `embedchain.llm`. E.g.: - -```python -from embedchain import App -from embedchain.llm.llama2 import Llama2Llm - -app = App(llm=Llama2Llm()) -``` - -#### Configuration - -The LLMs can be configured by passing an LlmConfig object. - -The config options can be found [here](/advanced/query_configuration#llmconfig) - -```python -from embedchain import App -from embedchain.llm.llama2 import Llama2Llm -from embedchain.config import LlmConfig - -app = App(llm=Llama2Llm(), llm_config=LlmConfig(number_documents=3, temperature=0)) -``` - -### Embedder - -#### Choosing an Embedder - -The following providers for embedding functions are supported by Embedchain: -- OPENAI -- HUGGING_FACE -- VERTEX_AI -- GPT4ALL -- AZURE_OPENAI - -You can choose one by importing it from `embedchain.embedder`. E.g.: - -```python -from embedchain import App -from embedchain.embedder.vertexai import VertexAiEmbedder - -app = App(embedder=VertexAiEmbedder()) -``` - -#### Configuration - -The LLMs can be configured by passing an EmbedderConfig object. - -```python -from embedchain import App -from embedchain.embedder.openai import OpenAiEmbedder -from embedchain.config import EmbedderConfig - -app = App(embedder=OpenAiEmbedder(), embedder_config=EmbedderConfig(model="text-embedding-ada-002")) -``` - - -You can also pass an `LlmConfig` instance directly to the `query` or `chat` method. -This creates a temporary config for that request alone, so you could, for example, use a different model (from the same provider) or get more context documents for a specific query. - - -### Vector Database - -#### Choosing a Vector Database - -The following vector databases are supported by Embedchain: -- ChromaDB -- Elasticsearch - -You can choose one by importing it from `embedchain.vectordb`. E.g.: - -```python -from embedchain import App -from embedchain.vectordb.elasticsearch import ElasticsearchDB - -app = App(db=ElasticsearchDB()) -``` - -#### Configuration - -The vector databases can be configured by passing a specific config object. - -These vary greatly between the different vector databases. - -```python -from embedchain import App -from embedchain.vectordb.elasticsearch import ElasticsearchDB -from embedchain.config import ElasticsearchDBConfig - -app = App(db=ElasticsearchDB(), db_config=ElasticsearchDBConfig()) -``` - -### PersonApp - -```python -from embedchain import PersonApp -naval_chat_bot = PersonApp("name_of_person_or_character") #Like "Yoda" -``` - -- `PersonApp` uses OpenAI's model, so these are paid models. πŸ’Έ You will be charged for embedding model usage and LLM usage. -- `PersonApp` uses OpenAI's embedding model to create embeddings for chunks and ChatGPT API as LLM to get answer given the relevant docs. Make sure that you have an OpenAI account and an API key. If you don't have an API key, you can create one by visiting [this link](https://platform.openai.com/account/api-keys). -- Once you have the API key, set it in an environment variable called `OPENAI_API_KEY` - -```python -import os -os.environ["OPENAI_API_KEY"] = "sk-xxxx" -``` - -### Full Configuration Examples - -Embedchain previously offered fully configured classes, namely `App`, `OpenSourceApp`, `CustomApp` and `Llama2App`. -We deprecated these apps. The reason for this decision was that it was hard to switch from to a different LLM, embedder or vector db, if you one day decided that that's what you want to do. -The new app allows drop-in replacements, such as changing `App(llm=OpenAiLlm())` to `App(llm=Llama2Llm())`. - -To make the switch to our new, fully configurable app easier, we provide you with full examples for what the old classes would look like implemented as a new app. -You can swap these in, and if you decide you want to try a different model one day, you don't have to rewrite your whole bot. - -#### App -App without any configuration is still using the best options available, so you can keep using: - -```python -from embedchain import App - -app = App() -``` - -#### OpenSourceApp - -Use this snippet to run an open source app. - -```python -from embedchain import App -from embedchain.llm.gpt4all import GPT4ALLLlm -from embedchain.embedder.gpt4all import GPT4AllEmbedder -from embedchain.vectordb.chroma import ChromaDB - -app = App(llm=GPT4ALLLlm(), embedder=GPT4AllEmbedder(), db=ChromaDB()) -``` - -#### Llama2App -```python -from embedchain import App -from embedchain.llm.llama2 import Llama2Llm - -app = App(llm=Llama2Llm()) -``` - -#### CustomApp - -Every app is a custom app now. -If you were previously using a `CustomApp`, you can now just change it to `App`. - -Here's one example, what you could do if we combined everything shown on this page. - -```python -from embedchain import App -from embedchain.config import ElasticsearchDBConfig, EmbedderConfig, LlmConfig -from embedchain.embedder.openai import OpenAiEmbedder -from embedchain.llm.llama2 import Llama2Llm -from embedchain.vectordb.elasticsearch import ElasticsearchDB - -app = App( - llm=Llama2Llm(), - llm_config=LlmConfig(number_documents=3, temperature=0), - embedder=OpenAiEmbedder(), - embedder_config=EmbedderConfig(model="text-embedding-ada-002"), - db=ElasticsearchDB(), - db_config=ElasticsearchDBConfig(), -) -``` - -### Compatibility with other apps - -- If there is any other app instance in your script or app, you can change the import as - -```python -from embedchain import App as EmbedChainApp -from embedchain import PersonApp as EmbedChainPersonApp - -# or - -from embedchain import App as ECApp -from embedchain import PersonApp as ECPApp -``` diff --git a/docs/advanced/configuration.mdx b/docs/advanced/configuration.mdx index dc6301c8..440384d1 100644 --- a/docs/advanced/configuration.mdx +++ b/docs/advanced/configuration.mdx @@ -4,101 +4,72 @@ title: 'βš™οΈ Custom configurations' Embedchain is made to work out of the box. However, for advanced users we're also offering configuration options. All of these configuration options are optional and have sane defaults. -## Concept -The main `App` class is available in the following varieties: `CustomApp`, `OpenSourceApp` and `Llama2App` and `App`. The first is fully configurable, the others are opinionated in some aspects. +You can configure different components of your app (`llm`, `embedding model`, or `vector database`) through a simple yaml configuration that Embedchain offers. Here is a generic full-stack example of the yaml config: -The `App` class has three subclasses: `llm`, `db` and `embedder`. These are the core ingredients that make up an EmbedChain app. -App plus each one of the subclasses have a `config` attribute. -You can pass a `Config` instance as an argument during initialization to persistently configure a class. -These configs can be imported from `embedchain.config` +```yaml +app: + config: + id: 'full-stack-app' -There are `set` methods for some things that should not (only) be set at start-up, like `app.db.set_collection_name`. +llm: + provider: openai + model: 'gpt-3.5-turbo' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 + stream: false + template: | + Use the following pieces of context to answer the query at the end. + If you don't know the answer, just say that you don't know, don't try to make up an answer. -## Examples + $context -### General + Query: $query -Here's the readme example with configuration options. + Helpful Answer: + system_prompt: | + Act as William Shakespeare. Answer the following questions in the style of William Shakespeare. -```python -from embedchain import App -from embedchain.config import AppConfig, AddConfig, LlmConfig, ChunkerConfig +vectordb: + provider: chroma + config: + collection_name: 'full-stack-app' + dir: db + allow_reset: true -# Example: set the log level for debugging -config = AppConfig(log_level="DEBUG") -naval_chat_bot = App(config) - -# Example: specify a custom collection name -naval_chat_bot.db.set_collection_name("naval_chat_bot") - -# Example: define your own chunker config for `youtube_video` -chunker_config = ChunkerConfig(chunk_size=1000, chunk_overlap=100, length_function=len) -# Example: Add your chunker config to an AddConfig to actually use it -add_config = AddConfig(chunker=chunker_config) -naval_chat_bot.add("https://www.youtube.com/watch?v=3qHkcs3kG44", config=add_config) - -# Example: Reset to default -add_config = AddConfig() -naval_chat_bot.add("https://navalmanack.s3.amazonaws.com/Eric-Jorgenson_The-Almanack-of-Naval-Ravikant_Final.pdf", config=add_config) -naval_chat_bot.add("https://nav.al/feedback", config=add_config) -naval_chat_bot.add("https://nav.al/agi", config=add_config) -naval_chat_bot.add(("Who is Naval Ravikant?", "Naval Ravikant is an Indian-American entrepreneur and investor."), config=add_config) - -# Change the number of documents. -query_config = LlmConfig(number_documents=5) -print(naval_chat_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?", config=query_config)) +embedder: + provider: openai + config: + model: 'text-embedding-ada-002' ``` -### Custom prompt template +Alright, let's dive into what each key means in the yaml config above: -Here's the example of using custom prompt template with `.query` +1. `app` Section: + - `config`: + - `id` (String): The ID or name of your full-stack application. +2. `llm` Section: + - `provider` (String): The provider for the language model, which is set to 'openai'. You can find the full list of llm providers in [our docs](/components/llms). + - `model` (String): The specific model being used, 'gpt-3.5-turbo'. + - `config`: + - `temperature` (Float): Controls the randomness of the model's output. A higher value (closer to 1) makes the output more random. + - `max_tokens` (Integer): Controls how many tokens are used in the response. + - `top_p` (Float): Controls the diversity of word selection. A higher value (closer to 1) makes word selection more diverse. + - `stream` (Boolean): Controls if the response is streamed back to the user (set to false). + - `template` (String): A custom template for the prompt that the model uses to generate responses. + - `system_prompt` (String): A system prompt for the model to follow when generating responses, in this case, it's set to the style of William Shakespeare. +3. `vectordb` Section: + - `provider` (String): The provider for the vector database, set to 'chroma'. You can find the full list of vector database providers in [our docs](/components/vector-databases). + - `config`: + - `collection_name` (String): The initial collection name for the database, set to 'full-stack-app'. + - `dir` (String): The directory for the database, set to 'db'. + - `allow_reset` (Boolean): Indicates whether resetting the database is allowed, set to true. +4. `embedder` Section: + - `provider` (String): The provider for the embedder, set to 'openai'. You can find the full list of embedding model providers in [our docs](/components/embedding-models). + - `config`: + - `model` (String): The specific model used for text embedding, 'text-embedding-ada-002'. -```python -from string import Template +If you have questions about the configuration above, please feel free to reach out to us using one of the following methods: -import wikipedia - -from embedchain import App -from embedchain.config import LlmConfig - -einstein_chat_bot = App() - -# Embed Wikipedia page -page = wikipedia.page("Albert Einstein") -einstein_chat_bot.add(page.content) - -# Example: use your own custom template with `$context` and `$query` -einstein_chat_template = Template( - """ - You are Albert Einstein, a German-born theoretical physicist, - widely ranked among the greatest and most influential scientists of all time. - - Use the following information about Albert Einstein to respond to - the human's query acting as Albert Einstein. - Context: $context - - Keep the response brief. If you don't know the answer, just say that you don't know, don't try to make up an answer. - - Human: $query - Albert Einstein:""" -) -# Example: Use the template, also add a system prompt. -llm_config = LlmConfig(template=einstein_chat_template, system_prompt="You are Albert Einstein.") -queries = [ - "Where did you complete your studies?", - "Why did you win nobel prize?", - "Why did you divorce your first wife?", -] -for query in queries: - response = einstein_chat_bot.query(query, config=llm_config) - print("Query: ", query) - print("Response: ", response) - -# Output -# Query: Where did you complete your studies? -# Response: I completed my secondary education at the Argovian cantonal school in Aarau, Switzerland. -# Query: Why did you win nobel prize? -# Response: I won the Nobel Prize in Physics in 1921 for my services to Theoretical Physics, particularly for my discovery of the law of the photoelectric effect. -# Query: Why did you divorce your first wife? -# Response: We divorced due to living apart for five years. -``` + \ No newline at end of file diff --git a/docs/advanced/interface_types.mdx b/docs/advanced/interface_types.mdx deleted file mode 100644 index 8096c6c1..00000000 --- a/docs/advanced/interface_types.mdx +++ /dev/null @@ -1,75 +0,0 @@ ---- -title: '🀝 Interface types' ---- - -## Interface Types - -The embedchain app exposes the following methods. - -### Query Interface - -- This interface is like a question answering bot. It takes a question and gets the answer. It does not maintain context about the previous chats.❓ - -- To use this, call `.query()` function to get the answer for any query. - -```python -print(naval_chat_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?")) -# answer: Naval argues that humans possess the unique capacity to understand explanations or concepts to the maximum extent possible in this physical reality. -``` - -### Chat Interface - -- This interface is a chat interface that remembers previous conversations. Right now it remembers 5 conversations by default. πŸ’¬ - -- To use this, call `.chat` function to get the answer for any query. - -```python -print(naval_chat_bot.chat("How to be happy in life?")) -# answer: The most important trick to being happy is to realize happiness is a skill you develop and a choice you make. You choose to be happy, and then you work at it. It's just like building muscles or succeeding at your job. It's about recognizing the abundance and gifts around you at all times. - -print(naval_chat_bot.chat("who is naval ravikant?")) -# answer: Naval Ravikant is an Indian-American entrepreneur and investor. - -print(naval_chat_bot.chat("what did the author say about happiness?")) -# answer: The author, Naval Ravikant, believes that happiness is a choice you make and a skill you develop. He compares the mind to the body, stating that just as the body can be molded and changed, so can the mind. He emphasizes the importance of being present in the moment and not getting caught up in regrets of the past or worries about the future. By being present and grateful for where you are, you can experience true happiness. -``` - -#### Dry Run - -Dry Run is an option in the `add`, `query` and `chat` methods that allows the user to display the data chunks and their constructed prompt which is not sent to the LLM, to save money. It's used for [testing](/advanced/testing#dry-run). - - -### Stream Response - -- You can add config to your query method to stream responses like ChatGPT does. You would require a downstream handler to render the chunk in your desirable format. Supports both OpenAI model and OpenSourceApp. πŸ“Š - -- To use this, instantiate a `LlmConfig` or `ChatConfig` object with `stream=True`. Then pass it to the `.chat()` or `.query()` method. The following example iterates through the chunks and prints them as they appear. - -```python -app = App() -query_config = LlmConfig(stream = True) -resp = app.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?", query_config) - -for chunk in resp: - print(chunk, end="", flush=True) -# answer: Naval argues that humans possess the unique capacity to understand explanations or concepts to the maximum extent possible in this physical reality. -``` - -### Other Methods - -#### Reset - -Resets the database and deletes all embeddings. Irreversible. Requires reinitialization afterwards. - -```python -app.reset() -``` - -#### Count - -Counts the number of embeddings (chunks) in the database. - -```python -print(app.db.count()) -# returns: 481 -``` diff --git a/docs/advanced/query_configuration.mdx b/docs/advanced/query_configuration.mdx index f3c39acb..f1c9c31c 100644 --- a/docs/advanced/query_configuration.mdx +++ b/docs/advanced/query_configuration.mdx @@ -49,11 +49,7 @@ Default values of chunker config parameters for different `data_type`: |docs_site|500|50|len| |notion|300|0|len| -### LoaderConfig - -_coming soon_ - -## LlmConfig +## BaseLlmConfig |option|description|type|default| |---|---|---|---| @@ -68,12 +64,3 @@ _coming soon_ |deployment_name|t.b.a.|str|None| |system_prompt|System prompt string. Unused if none.|str|None| |where|filter for context search.|dict|None| - - -## ChatConfig - -All options for query and... - -_coming soon_ - -`history` is not supported, as that is handled is handled automatically, the config option is not supported. diff --git a/docs/components/embedding-models.mdx b/docs/components/embedding-models.mdx new file mode 100644 index 00000000..9966e99f --- /dev/null +++ b/docs/components/embedding-models.mdx @@ -0,0 +1,135 @@ +--- +title: 🧩 Embedding models +--- + +## Overview + +Embedchain supports several embedding models from the following providers: + + + + + + + + +## OpenAI + +To use OpenAI embedding function, you have to set the `OPENAI_API_KEY` environment variable. You can obtain the OpenAI API key from the [OpenAI Platform](https://platform.openai.com/account/api-keys). + +Once you have obtained the key, you can use it like this: + + + +```python main.py +import os +from embedchain import App + +os.environ['OPENAI_API_KEY'] = 'xxx' + +# load embedding model configuration from openai.yaml file +app = App.from_config(yaml_path="openai.yaml") + +app.add("https://en.wikipedia.org/wiki/OpenAI") +app.query("What is OpenAI?") +``` + +```yaml openai.yaml +embedder: + provider: openai + config: + model: 'text-embedding-ada-002' +``` + + + +## GPT4ALL + +GPT4All supports generating high quality embeddings of arbitrary length documents of text using a CPU optimized contrastively trained Sentence Transformer. + + + +```python main.py +from embedchain import App + +# load embedding model configuration from gpt4all.yaml file +app = App.from_config(yaml_path="gpt4all.yaml") +``` + +```yaml gpt4all.yaml +llm: + provider: gpt4all + model: 'orca-mini-3b.ggmlv3.q4_0.bin' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 + stream: false + +embedder: + provider: gpt4all + config: + model: 'all-MiniLM-L6-v2' +``` + + + +## Hugging Face + +Hugging Face supports generating embeddings of arbitrary length documents of text using Sentence Transformer library. Example of how to generate embeddings using hugging face is given below: + + + +```python main.py +from embedchain import App + +# load embedding model configuration from huggingface.yaml file +app = App.from_config(yaml_path="huggingface.yaml") +``` + +```yaml huggingface.yaml +llm: + provider: huggingface + model: 'google/flan-t5-xxl' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 0.5 + stream: false + +embedder: + provider: huggingface + config: + model: 'sentence-transformers/all-mpnet-base-v2' +``` + + + +## Vertex AI + +Embedchain supports Google's VertexAI embeddings model through a simple interface. You just have to pass the `model_name` in the config yaml and it would work out of the box. + + + +```python main.py +from embedchain import App + +# load embedding model configuration from vertexai.yaml file +app = App.from_config(yaml_path="vertexai.yaml") +``` + +```yaml vertexai.yaml +llm: + provider: vertexai + model: 'chat-bison' + config: + temperature: 0.5 + top_p: 0.5 + +embedder: + provider: vertexai + config: + model: 'textembedding-gecko' +``` + + diff --git a/docs/components/llms.mdx b/docs/components/llms.mdx new file mode 100644 index 00000000..a4d72556 --- /dev/null +++ b/docs/components/llms.mdx @@ -0,0 +1,280 @@ +--- +title: πŸ€– Large language models (LLMs) +--- + +## Overview + +Embedchain comes with built-in support for various popular large language models. We handle the complexity of integrating these models for you, allowing you to easily customize your language model interactions through a user-friendly interface. + + + + + + + + + + + + + +## OpenAI + +To use OpenAI LLM models, you have to set the `OPENAI_API_KEY` environment variable. You can obtain the OpenAI API key from the [OpenAI Platform](https://platform.openai.com/account/api-keys). + +Once you have obtained the key, you can use it like this: + +```python +import os +from embedchain import App + +os.environ['OPENAI_API_KEY'] = 'xxx' + +app = App() +app.add("https://en.wikipedia.org/wiki/OpenAI") +app.query("What is OpenAI?") +``` + +If you are looking to configure the different parameters of the LLM, you can do so by loading the app using a [yaml config](https://github.com/embedchain/embedchain/blob/main/embedchain/yaml/chroma.yaml) file. + + + +```python main.py +import os +from embedchain import App + +os.environ['OPENAI_API_KEY'] = 'xxx' + +# load llm configuration from openai.yaml file +app = App.from_config(yaml_path="openai.yaml") +``` + +```yaml openai.yaml +llm: + provider: openai + model: 'gpt-3.5-turbo' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 + stream: false +``` + + + + +## Azure OpenAI + +_Coming soon_ + +## Anthropic + +To use anthropic's model, please set the `ANTHROPIC_API_KEY` which you find on their [Account Settings Page](https://console.anthropic.com/account/keys). + + + +```python main.py +import os +from embedchain import App + +os.environ["ANTHROPIC_API_KEY"] = "xxx" + +# load llm configuration from anthropic.yaml file +app = App.from_config(yaml_path="anthropic.yaml") +``` + +```yaml anthropic.yaml +llm: + provider: anthropic + model: 'claude-instant-1' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 + stream: false +``` + + + +
+ + +You may also have to set the `OPENAI_API_KEY` if you use the OpenAI's embedding model. + + + +## Cohere + +Set the `COHERE_API_KEY` as environment variable which you can find on their [Account settings page](https://dashboard.cohere.com/api-keys). + +Once you have the API key, you are all set to use it with Embedchain. + + + +```python main.py +import os +from embedchain import App + +os.environ["COHERE_API_KEY"] = "xxx" + +# load llm configuration from cohere.yaml file +app = App.from_config(yaml_path="cohere.yaml") +``` + +```yaml cohere.yaml +llm: + provider: cohere + model: large + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 +``` + + + +## GPT4ALL + +GPT4all is a free-to-use, locally running, privacy-aware chatbot. No GPU or internet required. You can use this with Embedchain using the following code: + + + +```python main.py +from embedchain import App + +# load llm configuration from gpt4all.yaml file +app = App.from_config(yaml_path="gpt4all.yaml") +``` + +```yaml gpt4all.yaml +llm: + provider: gpt4all + model: 'orca-mini-3b.ggmlv3.q4_0.bin' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 + stream: false + +embedder: + provider: gpt4all + config: + model: 'all-MiniLM-L6-v2' +``` + + + +## JinaChat + +First, set `JINACHAT_API_KEY` in environment variable which you can obtain from [their platform](https://chat.jina.ai/api). + +Once you have the key, load the app using the config yaml file: + + + +```python main.py +import os +from embedchain import App + +os.environ["JINACHAT_API_KEY"] = "xxx" +# load llm configuration from jina.yaml file +app = App.from_config(yaml_path="jina.yaml") +``` + +```yaml jina.yaml +llm: + provider: jina + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 + stream: false +``` + + + +## Hugging Face + +First, set `HUGGINGFACE_ACCESS_TOKEN` in environment variable which you can obtain from [their platform](https://huggingface.co/settings/tokens). + +Once you have the token, load the app using the config yaml file: + + + +```python main.py +import os +from embedchain import App + +os.environ["HUGGINGFACE_ACCESS_TOKEN"] = "xxx" + +# load llm configuration from huggingface.yaml file +app = App.from_config(yaml_path="huggingface.yaml") +``` + +```yaml huggingface.yaml +llm: + provider: huggingface + model: 'google/flan-t5-xxl' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 0.5 + stream: false +``` + + +## Llama2 + +Llama2 is integrated through [Replicate](https://replicate.com/). Set `REPLICATE_API_TOKEN` in environment variable which you can obtain from [their platform](https://replicate.com/account/api-tokens). + +Once you have the token, load the app using the config yaml file: + + + +```python main.py +import os +from embedchain import App + +os.environ["REPLICATE_API_TOKEN"] = "xxx" + +# load llm configuration from llama2.yaml file +app = App.from_config(yaml_path="llama2.yaml") +``` + +```yaml llama2.yaml +llm: + provider: llama2 + model: 'a16z-infra/llama13b-v2-chat:df7690f1994d94e96ad9d568eac121aecf50684a0b0963b25a41cc40061269e5' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 0.5 + stream: false +``` + + +## Vertex AI + +Setup Google Cloud Platform application credentials by following the instruction on [GCP](https://cloud.google.com/docs/authentication/external/set-up-adc). Once setup is done, use the following code to create an app using VertexAI as provider: + + + +```python main.py +from embedchain import App + +# load llm configuration from vertexai.yaml file +app = App.from_config(yaml_path="vertexai.yaml") +``` + +```yaml vertexai.yaml +llm: + provider: vertexai + model: 'chat-bison' + config: + temperature: 0.5 + top_p: 0.5 +``` + + +
+ diff --git a/docs/components/vector-databases.mdx b/docs/components/vector-databases.mdx new file mode 100644 index 00000000..dfcb5845 --- /dev/null +++ b/docs/components/vector-databases.mdx @@ -0,0 +1,142 @@ +--- +title: πŸ—„οΈ Vector databases +--- + +## Overview + +Utilizing a vector database alongside Embedchain is a seamless process. All you need to do is configure it within the YAML configuration file. We've provided examples for each supported database below: + + + + + + + + + + + + +## ChromaDB + + + +```python main.py +from embedchain import App + +# load chroma configuration from yaml file +app = App.from_config(yaml_path="chroma-config-1.yaml") +``` + +```yaml chroma-config-1.yaml +vectordb: + provider: chroma + config: + collection_name: 'my-collection' + dir: db + allow_reset: true +``` + +```yaml chroma-config-2.yaml +vectordb: + provider: chroma + config: + collection_name: 'my-collection' + host: localhost + port: 5200 + allow_reset: true +``` + + + + +## Elasticsearch + + + +```python main.py +from embedchain import App + +# load elasticsearch configuration from yaml file +app = App.from_config(yaml_path="elasticsearch.yaml") +``` + +```yaml elasticsearch.yaml +vectordb: + provider: elasticsearch + config: + collection_name: 'es-index' + es_url: http://localhost:9200 + allow_reset: true + api_key: xxx +``` + + +## OpenSearch + + + +```python main.py +from embedchain import App + +# load opensearch configuration from yaml file +app = App.from_config(yaml_path="opensearch.yaml") +``` + +```yaml opensearch.yaml +vectordb: + provider: opensearch + config: + opensearch_url: 'https://localhost:9200' + http_auth: + - admin + - admin + vector_dimension: 1536 + collection_name: 'my-app' + use_ssl: false + verify_certs: false +``` + + + +## Zilliz + + + +```python main.py +from embedchain import App + +# load zilliz configuration from yaml file +app = App.from_config(yaml_path="zilliz.yaml") +``` + +```yaml zilliz.yaml +vectordb: + provider: zilliz + config: + collection_name: 'zilliz-app' + uri: https://xxxx.api.gcp-region.zillizcloud.com + token: xxx + vector_dim: 1536 + metric_type: L2 +``` + + + +## LanceDB + +_Coming soon_ + +## Pinecone + +_Coming soon_ + +## Qdrant + +_Coming soon_ + +## Weaviate + +_Coming soon_ + + diff --git a/docs/data-sources/csv.mdx b/docs/data-sources/csv.mdx index 47111db2..b56453ec 100644 --- a/docs/data-sources/csv.mdx +++ b/docs/data-sources/csv.mdx @@ -1,14 +1,19 @@ --- -title: 'CSV' +title: 'πŸ“Š CSV' --- -### CSV file - To add any csv file, use the data_type as `csv`. `csv` allows remote urls and conventional file paths. Headers are included for each line, so if you have an `age` column, `18` will be added as `age: 18`. Eg: ```python -app.add('https://example.com/content/sheet.csv', data_type="csv") -app.add('content/sheet.csv', data_type="csv") +from embedchain import App + +app = App() +app.add('https://people.sc.fsu.edu/~jburkardt/data/csv/airtravel.csv', data_type="csv") +# Or add using the local file path +# app.add('/path/to/file.csv', data_type="csv") + +app.query("Summarize the air travel data") +# Answer: The air travel data shows the number of flights for the months of July in the years 1958, 1959, and 1960. In July 1958, there were 491 flights, in July 1959 there were 548 flights, and in July 1960 there were 622 flights. ``` -Note: There is a size limit allowed for csv file beyond which it can throw error. This limit is set by the LLMs. Please consider chunking large csv files into smaller csv files. \ No newline at end of file +Note: There is a size limit allowed for csv file beyond which it can throw error. This limit is set by the LLMs. Please consider chunking large csv files into smaller csv files. diff --git a/docs/data-sources/data-type-handling.mdx b/docs/data-sources/data-type-handling.mdx index 8861586d..bc2ed3dc 100644 --- a/docs/data-sources/data-type-handling.mdx +++ b/docs/data-sources/data-type-handling.mdx @@ -1,18 +1,16 @@ --- -title: 'Data Type Handling' +title: 'Data type handling' --- ## Automatic data type detection + The add method automatically tries to detect the data_type, based on your input for the source argument. So `app.add('https://www.youtube.com/watch?v=dQw4w9WgXcQ')` is enough to embed a YouTube video. This detection is implemented for all formats. It is based on factors such as whether it's a URL, a local file, the source data type, etc. ### Debugging automatic detection - -Set `log_level=DEBUG` (in [AppConfig](http://localhost:3000/advanced/query_configuration#appconfig)) and make sure it's working as intended. - -Otherwise, you will not know when, for instance, an invalid filepath is interpreted as raw text instead. +Set `log_level: DEBUG` in the config yaml to debug if the data type detection is done right or not. Otherwise, you will not know when, for instance, an invalid filepath is interpreted as raw text instead. ### Forcing a data type @@ -21,7 +19,7 @@ The examples below show you the keyword to force the respective `data_type`. Forcing can also be used for edge cases, such as interpreting a sitemap as a web_page, for reading its raw text instead of following links. -## Remote Data Types +## Remote data types **Use local files in remote data types** @@ -32,7 +30,7 @@ You can pass local files by formatting the path using the `file:` [URI scheme](h ## Reusing a vector database -Default behavior is to create a persistent vector DB in the directory **./db**. You can split your application into two Python scripts: one to create a local vector DB and the other to reuse this local persistent vector DB. This is useful when you want to index hundreds of documents and separately implement a chat interface. +Default behavior is to create a persistent vector db in the directory **./db**. You can split your application into two Python scripts: one to create a local vector db and the other to reuse this local persistent vector db. This is useful when you want to index hundreds of documents and separately implement a chat interface. Create a local index: diff --git a/docs/data-sources/docs-site.mdx b/docs/data-sources/docs-site.mdx index d3e23141..206ef209 100644 --- a/docs/data-sources/docs-site.mdx +++ b/docs/data-sources/docs-site.mdx @@ -1,11 +1,14 @@ --- -title: 'Code Documentation' +title: 'πŸ“šπŸŒ Code documentation' --- -### Code documentation - To add any code documentation website as a loader, use the data_type as `docs_site`. Eg: ```python +from embedchain import App + +app = App() app.add("https://docs.embedchain.ai/", data_type="docs_site") -``` \ No newline at end of file +app.query("What is Embedchain?") +# Answer: Embedchain is a platform that utilizes various components, including paid/proprietary ones, to provide what is believed to be the best configuration available. It uses LLM (Language Model) providers such as OpenAI, Anthpropic, Vertex_AI, GPT4ALL, Azure_OpenAI, LLAMA2, JINA, and COHERE. Embedchain allows users to import and utilize these LLM providers for their applications.' +``` diff --git a/docs/data-sources/docx.mdx b/docs/data-sources/docx.mdx index c5eeb6b0..cc459621 100644 --- a/docs/data-sources/docx.mdx +++ b/docs/data-sources/docx.mdx @@ -1,5 +1,5 @@ --- -title: 'Docx File' +title: 'πŸ“„ Docx file' --- ### Docx file @@ -7,6 +7,12 @@ title: 'Docx File' To add any doc/docx file, use the data_type as `docx`. `docx` allows remote urls and conventional file paths. Eg: ```python +from embedchain import App + +app = App() app.add('https://example.com/content/intro.docx', data_type="docx") -app.add('content/intro.docx', data_type="docx") -``` \ No newline at end of file +# Or add file using the local file path on your system +# app.add('content/intro.docx', data_type="docx") + +app.query("Summarize the docx data?") +``` diff --git a/docs/data-sources/how-to-add-data.mdx b/docs/data-sources/how-to-add-data.mdx deleted file mode 100644 index 699c2d15..00000000 --- a/docs/data-sources/how-to-add-data.mdx +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: 'How to add data' ---- - -## Add Dataset - -- This step assumes that you have already created an `App`. We are calling our app instance as `naval_chat_bot` πŸ€– - -- Now use `.add` method to add any dataset. - -```python -naval_chat_bot = App() - -# Embed Online Resources -naval_chat_bot.add("https://www.youtube.com/watch?v=3qHkcs3kG44") -naval_chat_bot.add("https://navalmanack.s3.amazonaws.com/Eric-Jorgenson_The-Almanack-of-Naval-Ravikant_Final.pdf") -naval_chat_bot.add("https://nav.al/feedback") -naval_chat_bot.add("https://nav.al/agi") - -# Embed Local Resources -naval_chat_bot.add(("Who is Naval Ravikant?", "Naval Ravikant is an Indian-American entrepreneur and investor.")) -``` - -The possible formats to add data can be found on the [Supported Data Formats](/advanced/data_types) page. diff --git a/docs/data-sources/mdx.mdx b/docs/data-sources/mdx.mdx index 02681c68..c59569e5 100644 --- a/docs/data-sources/mdx.mdx +++ b/docs/data-sources/mdx.mdx @@ -1,12 +1,14 @@ --- -title: 'Mdx' +title: 'πŸ“ Mdx file' --- - -### Mdx file - -To add any mdx file to your app, use the data_type (first argument to `.add()` method) as `mdx`. Note that this supports support mdx file present on machine, so this should be a file path. Eg: +To add any `.mdx` file to your app, use the data_type (first argument to `.add()` method) as `mdx`. Note that this supports support mdx file present on machine, so this should be a file path. Eg: ```python +from embedchain import App + +app = App() app.add('path/to/file.mdx', data_type='mdx') -``` \ No newline at end of file + +app.query("What are the docs about?") +``` diff --git a/docs/data-sources/notion.mdx b/docs/data-sources/notion.mdx index 9530dbf8..faa52de0 100644 --- a/docs/data-sources/notion.mdx +++ b/docs/data-sources/notion.mdx @@ -1,15 +1,20 @@ --- -title: 'Notion' +title: 'πŸ““ Notion' --- -### Notion To use notion you must install the extra dependencies with `pip install --upgrade embedchain[notion]`. -To load a notion page, use the data_type as `notion`. Since it is hard to automatically detect, forcing this is advised. +To load a notion page, use the data_type as `notion`. Since it is hard to automatically detect, it is advised to specify the `data_type` when adding a notion document. The next argument must **end** with the `notion page id`. The id is a 32-character string. Eg: ```python -app.add("cfbc134ca6464fc980d0391613959196", "notion") -app.add("my-page-cfbc134ca6464fc980d0391613959196", "notion") -app.add("https://www.notion.so/my-page-cfbc134ca6464fc980d0391613959196", "notion") -``` \ No newline at end of file +from embedchain import App + +app = App() + +app.add("cfbc134ca6464fc980d0391613959196", data_type="notion") +app.add("my-page-cfbc134ca6464fc980d0391613959196", data_type="notion") +app.add("https://www.notion.so/my-page-cfbc134ca6464fc980d0391613959196", data_type="notion") + +app.query("Summarize the notion doc") +``` diff --git a/docs/data-sources/overview.mdx b/docs/data-sources/overview.mdx new file mode 100644 index 00000000..c5372ce5 --- /dev/null +++ b/docs/data-sources/overview.mdx @@ -0,0 +1,24 @@ +--- +title: Overview +--- + +Embedchain comes with built-in support for various data sources. We handle the complexity of loading unstructured data from these data sources, allowing you to easily customize your app through a user-friendly interface. + + + + + + + + + + + + + + + + +
+ + diff --git a/docs/data-sources/pdf-file.mdx b/docs/data-sources/pdf-file.mdx index 028c3d56..fe8b8884 100644 --- a/docs/data-sources/pdf-file.mdx +++ b/docs/data-sources/pdf-file.mdx @@ -1,14 +1,17 @@ --- -title: 'PDF File' +title: 'πŸ“° PDF file' --- - -### PDF File - To add any pdf file, use the data_type as `pdf_file`. Eg: ```python -app.add('a_valid_url_where_pdf_file_can_be_accessed', data_type='pdf_file') +from embedchain import App + +app = App() + +app.add('https://arxiv.org/pdf/1706.03762.pdf', data_type='pdf_file') +app.query("What is the paper 'attention is all you need' about?") +# Answer: The paper "Attention Is All You Need" proposes a new network architecture called the Transformer, which is based solely on attention mechanisms. It suggests moving away from complex recurrent or convolutional neural networks and instead using attention mechanisms to connect the encoder and decoder in sequence transduction models. ``` Note that we do not support password protected pdfs. \ No newline at end of file diff --git a/docs/data-sources/qna.mdx b/docs/data-sources/qna.mdx index 87f118a5..f7a034d9 100644 --- a/docs/data-sources/qna.mdx +++ b/docs/data-sources/qna.mdx @@ -1,11 +1,13 @@ --- -title: 'QnA Pair' +title: 'β“πŸ’¬ Queston and answer pair' --- -### QnA pair - QnA pair is a local data type. To supply your own QnA pair, use the data_type as `qna_pair` and enter a tuple. Eg: ```python +from embedchain import App + +app = App() + app.add(("Question", "Answer"), data_type="qna_pair") ``` \ No newline at end of file diff --git a/docs/data-sources/request-new-format.mdx b/docs/data-sources/request-new-format.mdx deleted file mode 100644 index 824f2018..00000000 --- a/docs/data-sources/request-new-format.mdx +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: 'Request New Format' -url: https://forms.gle/gB5La14tjgy4p94dA ---- diff --git a/docs/data-sources/sitemap.mdx b/docs/data-sources/sitemap.mdx index d7ce7338..96b47ef1 100644 --- a/docs/data-sources/sitemap.mdx +++ b/docs/data-sources/sitemap.mdx @@ -1,11 +1,13 @@ --- -title: 'Sitemap' +title: 'πŸ—ΊοΈ Sitemap' --- -### Sitemap - Add all web pages from an xml-sitemap. Filters non-text files. Use the data_type as `sitemap`. Eg: ```python +from embedchain import App + +app = App() + app.add('https://example.com/sitemap.xml', data_type='sitemap') ``` \ No newline at end of file diff --git a/docs/data-sources/text.mdx b/docs/data-sources/text.mdx index 1ce81d2e..0fda6f57 100644 --- a/docs/data-sources/text.mdx +++ b/docs/data-sources/text.mdx @@ -1,5 +1,5 @@ --- -title: 'Text' +title: 'πŸ“ Text' --- ### Text @@ -7,7 +7,11 @@ title: 'Text' Text is a local data type. To supply your own text, use the data_type as `text` and enter a string. The text is not processed, this can be very versatile. Eg: ```python +from embedchain import App + +app = App() + app.add('Seek wealth, not money or status. Wealth is having assets that earn while you sleep. Money is how we transfer time and wealth. Status is your place in the social hierarchy.', data_type='text') ``` -Note: This is not used in the examples because in most cases you will supply a whole paragraph or file, which did not fit. \ No newline at end of file +Note: This is not used in the examples because in most cases you will supply a whole paragraph or file, which did not fit. diff --git a/docs/data-sources/web-page.mdx b/docs/data-sources/web-page.mdx index e788bcd9..a30b836c 100644 --- a/docs/data-sources/web-page.mdx +++ b/docs/data-sources/web-page.mdx @@ -1,11 +1,13 @@ --- -title: 'Web page' +title: 'πŸŒπŸ“„ Web page' --- -### Web page - To add any web page, use the data_type as `web_page`. Eg: ```python +from embedchain import App + +app = App() + app.add('a_valid_web_page_url', data_type='web_page') -``` \ No newline at end of file +``` diff --git a/docs/data-sources/xml.mdx b/docs/data-sources/xml.mdx index 0398afdc..afe9a412 100644 --- a/docs/data-sources/xml.mdx +++ b/docs/data-sources/xml.mdx @@ -1,5 +1,5 @@ --- -title: 'XML File' +title: '🧾 XML file' --- ### XML file @@ -7,7 +7,11 @@ title: 'XML File' To add any xml file, use the data_type as `xml`. Eg: ```python +from embedchain import App + +app = App() + app.add('content/data.xml') ``` -Note: Only the text content of the xml file will be added to the app. The tags will be ignored. \ No newline at end of file +Note: Only the text content of the xml file will be added to the app. The tags will be ignored. diff --git a/docs/data-sources/youtube-video.mdx b/docs/data-sources/youtube-video.mdx index 82dea222..62a769ce 100644 --- a/docs/data-sources/youtube-video.mdx +++ b/docs/data-sources/youtube-video.mdx @@ -1,12 +1,13 @@ --- -title: 'Youtube Video' +title: 'πŸŽ₯πŸ“Ί Youtube video' --- -### Youtube video - To add any youtube video to your app, use the data_type (first argument to `.add()` method) as `youtube_video`. Eg: ```python +from embedchain import App + +app = App() app.add('a_valid_youtube_url_here', data_type='youtube_video') ``` diff --git a/docs/get-start/faq.mdx b/docs/get-start/faq.mdx deleted file mode 100644 index de95fab7..00000000 --- a/docs/get-start/faq.mdx +++ /dev/null @@ -1,16 +0,0 @@ ---- -title: ❓ Frequently Asked Questions -description: 'Collections of all the frequently asked questions about Embedchain' ---- - -## How to use GPT-4 as the LLM model - -```python - -from embedchain import App -from embedchain.config import LlmConfig - -app = App() -app.add("https://en.wikipedia.org/wiki/Elon_Musk") -app.query("How many companies does Elon Musk run and name those?", config=LlmConfig(model="gpt-4")) -``` \ No newline at end of file diff --git a/docs/get-start/introduction.mdx b/docs/get-start/introduction.mdx deleted file mode 100644 index 5aef48dd..00000000 --- a/docs/get-start/introduction.mdx +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: πŸ“š Introduction -description: 'πŸ“ Embedchain is a framework to easily create LLM powered bots over any dataset.' ---- - -## πŸ€” What is Embedchain? - -Embedchain abstracts the entire process of loading a dataset, chunking it, creating embeddings, and storing it in a vector database. - -You can add a single or multiple datasets using the `.add` method. Then, simply use the `.query` method to find answers from the added datasets. - -If you want to create a Naval Ravikant bot with a YouTube video, a book in PDF format, two blog posts, and a question and answer pair, all you need to do is add the respective links. Embedchain will take care of the rest, creating a bot for you. - -```python -from embedchain import App - -naval_chat_bot = App() -# Embed Online Resources -naval_chat_bot.add("https://www.youtube.com/watch?v=3qHkcs3kG44") -naval_chat_bot.add("https://navalmanack.s3.amazonaws.com/Eric-Jorgenson_The-Almanack-of-Naval-Ravikant_Final.pdf") -naval_chat_bot.add("https://nav.al/feedback") -naval_chat_bot.add("https://nav.al/agi") -naval_chat_bot.add("The Meanings of Life", 'text', metadata={'chapter': 'philosphy'}) - -# Embed Local Resources -naval_chat_bot.add(("Who is Naval Ravikant?", "Naval Ravikant is an Indian-American entrepreneur and investor.")) - -naval_chat_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?") -# Answer: Naval argues that humans possess the unique capacity to understand explanations or concepts to the maximum extent possible in this physical reality. - -# with where context filter -naval_chat_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?", where={'chapter': 'philosophy'}) -``` - -## πŸš€ How it works? - -Creating a chat bot over any dataset involves the following steps: - -1. Detect the data type and load the data -2. Create meaningful chunks -3. Create embeddings for each chunk -4. Store the chunks in a vector database - -When a user asks a query, the following process happens to find the answer: - -1. Create an embedding for the query -2. Find similar documents for the query from the vector database -3. Pass the similar documents as context to LLM to get the final answer. - -The process of loading the dataset and querying involves multiple steps, each with its own nuances: - -- How should I chunk the data? What is a meaningful chunk size? -- How should I create embeddings for each chunk? Which embedding model should I use? -- How should I store the chunks in a vector database? Which vector database should I use? -- Should I store metadata along with the embeddings? -- How should I find similar documents for a query? Which ranking model should I use? - -Embedchain takes care of all these nuances and provides a simple interface to create bots over any dataset. - -In the first release, we make it easier for anyone to get a chatbot over any dataset up and running in less than a minute. Just create an app instance, add the datasets using the `.add` method, and use the `.query` method to get the relevant answers. diff --git a/docs/get-start/quickstart.mdx b/docs/get-start/quickstart.mdx deleted file mode 100644 index 48ca3bd3..00000000 --- a/docs/get-start/quickstart.mdx +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: 'πŸš€ Quickstart' -description: 'πŸ’‘ Start building LLM powered bots under 30 seconds' ---- - -Install embedchain python package: - -```bash -pip install --upgrade embedchain -``` - -Creating a chatbot involves 3 steps: - -- βš™οΈ Import the App instance -- πŸ—ƒοΈ Add Dataset -- πŸ’¬ Query or Chat on the dataset and get answers (Interface Types) - -Run your first bot in python using the following code. Make sure to set the `OPENAI_API_KEY` πŸ”‘ environment variable in the code. - -```python -import os - -from embedchain import App - -os.environ["OPENAI_API_KEY"] = "xxx" -elon_musk_bot = App() - -# Embed Online Resources -elon_musk_bot.add("https://en.wikipedia.org/wiki/Elon_Musk") -elon_musk_bot.add("https://www.forbes.com/profile/elon-musk") - -response = elon_musk_bot.query("How many companies does Elon Musk run and name those?") -print(response) -# Answer: 'Elon Musk currently runs several companies. As of my knowledge, he is the CEO and lead designer of SpaceX, the CEO and product architect of Tesla, Inc., the CEO and founder of Neuralink, and the CEO and founder of The Boring Company. However, please note that this information may change over time, so it's always good to verify the latest updates.' -``` diff --git a/docs/get-started/faq.mdx b/docs/get-started/faq.mdx new file mode 100644 index 00000000..cfe84990 --- /dev/null +++ b/docs/get-started/faq.mdx @@ -0,0 +1,68 @@ +--- +title: ❓ FAQs +description: 'Collections of all the frequently asked questions' +--- + +#### How to use GPT-4 as the LLM model? + + + +```python main.py +import os +from embedchain import App + +os.environ['OPENAI_API_KEY'] = 'xxx' + +# load llm configuration from gpt4.yaml file +app = App.from_config(yaml_path="gpt4.yaml") +``` + +```yaml gpt4.yaml +llm: + provider: openai + model: 'gpt-4' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 + stream: false +``` + + + +#### I don't have OpenAI credits. How can I use some open source model? + + + +```python main.py +import os +from embedchain import App + +os.environ['OPENAI_API_KEY'] = 'xxx' + +# load llm configuration from opensource.yaml file +app = App.from_config(yaml_path="opensource.yaml") +``` + +```yaml opensource.yaml +llm: + provider: gpt4all + model: 'orca-mini-3b.ggmlv3.q4_0.bin' + config: + temperature: 0.5 + max_tokens: 1000 + top_p: 1 + stream: false + +embedder: + provider: gpt4all + config: + model: 'all-MiniLM-L6-v2' +``` + + +#### How to contact support? + +If docs aren't sufficient, please feel free to reach out to us using one of the following methods: + + diff --git a/docs/get-started/introduction.mdx b/docs/get-started/introduction.mdx new file mode 100644 index 00000000..422b4633 --- /dev/null +++ b/docs/get-started/introduction.mdx @@ -0,0 +1,58 @@ +--- +title: πŸ“š Introduction +description: 'πŸ“ Embedchain is a framework to easily create LLM powered apps on your data.' +--- + +## πŸ€” What is Embedchain? + +Embedchain abstracts the entire process of loading data, chunking it, creating embeddings, and storing it in a vector database. + +You can add data from different data sources using the `.add()` method. Then, simply use the `.query()` method to find answers from the added datasets. + +If you want to create a Naval Ravikant bot with a YouTube video, a book in PDF format, two blog posts, and a question and answer pair, all you need to do is add the respective links. Embedchain will take care of the rest, creating a bot for you. + +```python +from embedchain import App + +naval_bot = App() +# Add online data +naval_bot.add("https://www.youtube.com/watch?v=3qHkcs3kG44") +naval_bot.add("https://navalmanack.s3.amazonaws.com/Eric-Jorgenson_The-Almanack-of-Naval-Ravikant_Final.pdf") +naval_bot.add("https://nav.al/feedback") +naval_bot.add("https://nav.al/agi") +naval_bot.add("The Meanings of Life", 'text', metadata={'chapter': 'philosphy'}) + +# Add local resources +naval_bot.add(("Who is Naval Ravikant?", "Naval Ravikant is an Indian-American entrepreneur and investor.")) + +naval_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?") +# Answer: Naval argues that humans possess the unique capacity to understand explanations or concepts to the maximum extent possible in this physical reality. + +# Ask questions with specific context +naval_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?", where={'chapter': 'philosophy'}) +``` + +## πŸš€ How it works? + +Embedchain abstracts out the following steps from you to easily create LLM powered apps: + +1. Detect the data type and load data +2. Create meaningful chunks +3. Create embeddings for each chunk +4. Store chunks in a vector database + +When a user asks a query, the following process happens to find the answer: + +1. Create an embedding for the query +2. Find similar documents for the query from the vector database +3. Pass the similar documents as context to LLM to get the final answer + +The process of loading the dataset and querying involves multiple steps, each with its own nuances: + +- How should I chunk the data? What is a meaningful chunk size? +- How should I create embeddings for each chunk? Which embedding model should I use? +- How should I store the chunks in a vector database? Which vector database should I use? +- Should I store metadata along with the embeddings? +- How should I find similar documents for a query? Which ranking model should I use? + +Embedchain takes care of all these nuances and provides a simple interface to create apps on any data. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx new file mode 100644 index 00000000..71d37bfe --- /dev/null +++ b/docs/get-started/quickstart.mdx @@ -0,0 +1,52 @@ +--- +title: 'πŸš€ Quickstart' +description: 'πŸ’‘ Start building LLM powered apps under 30 seconds' +--- + +Install embedchain python package: + +```bash +pip install embedchain +``` + +Creating an app involves 3 steps: + + + +```python +from embedchain import App +app = App() +``` + + +```python +# Embed online resources +elon_bot.add("https://en.wikipedia.org/wiki/Elon_Musk") +elon_bot.add("https://www.forbes.com/profile/elon-musk") +``` + + +```python +elon_bot.query("What is the net worth of Elon Musk today?") +# Answer: The net worth of Elon Musk today is $258.7 billion. +``` + + + +Putting it together, you can run your first app using the following code. Make sure to set the `OPENAI_API_KEY` πŸ”‘ environment variable in the code. + +```python +import os +from embedchain import App + +os.environ["OPENAI_API_KEY"] = "xxx" +elon_bot = App() + +# Embed online resources +elon_bot.add("https://en.wikipedia.org/wiki/Elon_Musk") +elon_bot.add("https://www.forbes.com/profile/elon-musk") + +response = elon_bot.query("What is the net worth of Elon Musk today?") +print(response) +# Answer: The net worth of Elon Musk today is $258.7 billion. +``` diff --git a/docs/mint.json b/docs/mint.json index f8c1d4f7..e9787741 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -16,13 +16,13 @@ "name": "Twitter", "url": "https://twitter.com/embedchain" }, - { - "name": "Discord", - "url": "https://discord.gg/6PzXDgEjG5" - }, { "name":"Slack", "url":"https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw" + }, + { + "name": "Discord", + "url": "https://discord.gg/6PzXDgEjG5" } ], "topbarCtaButton": { @@ -32,20 +32,18 @@ "navigation": [ { "group": "Get started", - "pages": ["get-start/quickstart", "get-start/introduction", "get-start/faq"] - }, - - { - "group": "Advanced", - "pages": ["advanced/app_types", "advanced/interface_types", "advanced/query_configuration", "advanced/configuration", "advanced/testing", "advanced/vector_database"] + "pages": ["get-started/quickstart", "get-started/introduction", "get-started/faq"] }, { - "group": "Data Sources", + "group": "Components", + "pages": ["components/llms", "components/embedding-models", "components/vector-databases"] + }, + { + "group": "Data sources", "pages": [ - "data-sources/how-to-add-data", - "data-sources/data-type-handling", + "data-sources/overview", { - "group": "Supported Data Sources", + "group": "Supported data sources", "pages": [ "data-sources/csv", "data-sources/docs-site", @@ -60,9 +58,13 @@ "data-sources/youtube-video" ] }, - "data-sources/request-new-format" + "data-sources/data-type-handling" ] }, + { + "group": "Advanced", + "pages": ["advanced/configuration"] + }, { "group": "Examples", "pages": ["examples/full_stack", "examples/api_server", "examples/discord_bot", "examples/slack_bot", "examples/telegram_bot", "examples/whatsapp_bot", "examples/poe_bot"] @@ -75,7 +77,7 @@ ] }, { - "group": "Integration", + "group": "Integrations", "pages": ["integration/langsmith"] }, { @@ -98,12 +100,14 @@ ], "footerSocials": { - "twitter": "https://twitter.com/embedchain", - "github": "https://github.com/embedchain/embedchain", - "linkedin": "https://www.linkedin.com/company/embedchain", "website": "https://embedchain.ai", - "slack":"https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw" + "github": "https://github.com/embedchain/embedchain", + "slack":"https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw", + "discord": "https://discord.gg/6PzXDgEjG5", + "twitter": "https://twitter.com/embedchain", + "linkedin": "https://www.linkedin.com/company/embedchain" }, "backgroundImage": "/background.png", - "isWhiteLabeled": true + "isWhiteLabeled": true, + "feedback.thumbsRating": true } diff --git a/embedchain/apps/app.py b/embedchain/apps/app.py index 18f03c8e..8d96d2f1 100644 --- a/embedchain/apps/app.py +++ b/embedchain/apps/app.py @@ -46,7 +46,7 @@ class App(EmbedChain): :param llm: LLM Class instance. example: `from embedchain.llm.openai import OpenAILlm`, defaults to OpenAiLlm :type llm: BaseLlm, optional :param llm_config: Allows you to configure the LLM, e.g. how many documents to return, - example: `from embedchain.config import LlmConfig`, defaults to None + example: `from embedchain.config import BaseLlmConfig`, defaults to None :type llm_config: Optional[BaseLlmConfig], optional :param db: The database to use for storing and retrieving embeddings, example: `from embedchain.vectordb.chroma_db import ChromaDb`, defaults to ChromaDb diff --git a/embedchain/apps/open_source_app.py b/embedchain/apps/open_source_app.py index 194b2433..de32d59e 100644 --- a/embedchain/apps/open_source_app.py +++ b/embedchain/apps/open_source_app.py @@ -46,7 +46,7 @@ class OpenSourceApp(App): that does not fall into the LLM, database or embedder category, defaults to None :type config: OpenSourceAppConfig, optional :param llm_config: Allows you to configure the LLM, e.g. how many documents to return. - example: `from embedchain.config import LlmConfig`, defaults to None + example: `from embedchain.config import BaseLlmConfig`, defaults to None :type llm_config: BaseLlmConfig, optional :param chromadb_config: Allows you to configure the open source database, example: `from embedchain.config import ChromaDbConfig`, defaults to None @@ -54,7 +54,7 @@ class OpenSourceApp(App): :param system_prompt: System prompt that will be provided to the LLM as such. Please don't use for the time being, as it's not supported., defaults to None :type system_prompt: Optional[str], optional - :raises TypeError: `OpenSourceAppConfig` or `LlmConfig` invalid. + :raises TypeError: `OpenSourceAppConfig` or `BaseLlmConfig` invalid. """ logging.warning( "DEPRECATION WARNING: Please use `App` instead of `OpenSourceApp`." diff --git a/embedchain/apps/person_app.py b/embedchain/apps/person_app.py index 6c727482..02940def 100644 --- a/embedchain/apps/person_app.py +++ b/embedchain/apps/person_app.py @@ -4,8 +4,8 @@ from embedchain.apps.app import App from embedchain.apps.open_source_app import OpenSourceApp from embedchain.config import BaseLlmConfig from embedchain.config.apps.base_app_config import BaseAppConfig -from embedchain.config.llm.base_llm_config import (DEFAULT_PROMPT, - DEFAULT_PROMPT_WITH_HISTORY) +from embedchain.config.llm.base import (DEFAULT_PROMPT, + DEFAULT_PROMPT_WITH_HISTORY) from embedchain.helper.json_serializable import register_deserializable diff --git a/embedchain/bots/base.py b/embedchain/bots/base.py index 03279857..41d5ee96 100644 --- a/embedchain/bots/base.py +++ b/embedchain/bots/base.py @@ -1,7 +1,7 @@ from typing import Any from embedchain import App -from embedchain.config import AddConfig, AppConfig, LlmConfig +from embedchain.config import AddConfig, AppConfig, BaseLlmConfig from embedchain.embedder.openai import OpenAIEmbedder from embedchain.helper.json_serializable import (JSONSerializable, register_deserializable) @@ -27,14 +27,14 @@ class BaseBot(JSONSerializable): config = config if config else AddConfig() self.app.add(data, config=config) - def query(self, query: str, config: LlmConfig = None) -> str: + def query(self, query: str, config: BaseLlmConfig = None) -> str: """ Query the bot :param query: the user query :type query: str :param config: configuration class instance, defaults to None - :type config: LlmConfig, optional + :type config: BaseLlmConfig, optional :return: Answer :rtype: str """ diff --git a/embedchain/config/__init__.py b/embedchain/config/__init__.py index f9fc14ae..c5b3a247 100644 --- a/embedchain/config/__init__.py +++ b/embedchain/config/__init__.py @@ -7,8 +7,7 @@ from .apps.open_source_app_config import OpenSourceAppConfig from .base_config import BaseConfig from .embedder.base import BaseEmbedderConfig from .embedder.base import BaseEmbedderConfig as EmbedderConfig -from .llm.base_llm_config import BaseLlmConfig -from .llm.base_llm_config import BaseLlmConfig as LlmConfig +from .llm.base import BaseLlmConfig from .vectordb.chroma import ChromaDbConfig from .vectordb.elasticsearch import ElasticsearchDBConfig from .vectordb.opensearch import OpenSearchDBConfig diff --git a/embedchain/config/llm/base_llm_config.py b/embedchain/config/llm/base.py similarity index 98% rename from embedchain/config/llm/base_llm_config.py rename to embedchain/config/llm/base.py index e9c11515..a98c1b6d 100644 --- a/embedchain/config/llm/base_llm_config.py +++ b/embedchain/config/llm/base.py @@ -73,7 +73,6 @@ class BaseLlmConfig(BaseConfig): Initializes a configuration class instance for the LLM. Takes the place of the former `QueryConfig` or `ChatConfig`. - Use `LlmConfig` as an alias to `BaseLlmConfig`. :param number_documents: Number of documents to pull from the database as context, defaults to 1 @@ -115,6 +114,9 @@ class BaseLlmConfig(BaseConfig): self.system_prompt = system_prompt self.query_type = query_type + if type(template) is str: + template = Template(template) + if self.validate_template(template): self.template = template else: diff --git a/embedchain/embedchain.py b/embedchain/embedchain.py index dfbf3e66..ccb61511 100644 --- a/embedchain/embedchain.py +++ b/embedchain/embedchain.py @@ -470,7 +470,7 @@ class EmbedChain(JSONSerializable): :param input_query: The query to use. :type input_query: str - :param config: The `LlmConfig` instance to use as configuration options. This is used for one method call. + :param config: The `BaseLlmConfig` instance to use as configuration options. This is used for one method call. To persistently use a config, declare it during app init., defaults to None :type config: Optional[BaseLlmConfig], optional :param dry_run: A dry run does everything except send the resulting prompt to @@ -506,7 +506,7 @@ class EmbedChain(JSONSerializable): :param input_query: The query to use. :type input_query: str - :param config: The `LlmConfig` instance to use as configuration options. This is used for one method call. + :param config: The `BaseLlmConfig` instance to use as configuration options. This is used for one method call. To persistently use a config, declare it during app init., defaults to None :type config: Optional[BaseLlmConfig], optional :param dry_run: A dry run does everything except send the resulting prompt to diff --git a/embedchain/embedder/vertexai.py b/embedchain/embedder/vertexai.py index af842949..fe947646 100644 --- a/embedchain/embedder/vertexai.py +++ b/embedchain/embedder/vertexai.py @@ -7,7 +7,7 @@ from embedchain.embedder.base import BaseEmbedder from embedchain.models import VectorDimensions -class VertexAiEmbedder(BaseEmbedder): +class VertexAIEmbedder(BaseEmbedder): def __init__(self, config: Optional[BaseEmbedderConfig] = None): super().__init__(config=config) diff --git a/embedchain/factory.py b/embedchain/factory.py index 900695b8..79663962 100644 --- a/embedchain/factory.py +++ b/embedchain/factory.py @@ -13,16 +13,16 @@ class LlmFactory: "azure_openai": "embedchain.llm.azure_openai.AzureOpenAILlm", "cohere": "embedchain.llm.cohere.CohereLlm", "gpt4all": "embedchain.llm.gpt4all.GPT4ALLLlm", - "hugging_face_llm": "embedchain.llm.hugging_face_llm.HuggingFaceLlm", + "huggingface": "embedchain.llm.huggingface.HuggingFaceLlm", "jina": "embedchain.llm.jina.JinaLlm", "llama2": "embedchain.llm.llama2.Llama2Llm", "openai": "embedchain.llm.openai.OpenAILlm", "vertexai": "embedchain.llm.vertex_ai.VertexAILlm", } provider_to_config_class = { - "embedchain": "embedchain.config.llm.base_llm_config.BaseLlmConfig", - "openai": "embedchain.config.llm.base_llm_config.BaseLlmConfig", - "anthropic": "embedchain.config.llm.base_llm_config.BaseLlmConfig", + "embedchain": "embedchain.config.llm.base.BaseLlmConfig", + "openai": "embedchain.config.llm.base.BaseLlmConfig", + "anthropic": "embedchain.config.llm.base.BaseLlmConfig", } @classmethod @@ -43,7 +43,7 @@ class EmbedderFactory: provider_to_class = { "gpt4all": "embedchain.embedder.gpt4all.GPT4AllEmbedder", "huggingface": "embedchain.embedder.huggingface.HuggingFaceEmbedder", - "vertexai": "embedchain.embedder.vertexai.VertexAiEmbedder", + "vertexai": "embedchain.embedder.vertexai.VertexAIEmbedder", "openai": "embedchain.embedder.openai.OpenAIEmbedder", } provider_to_config_class = { diff --git a/embedchain/llm/base.py b/embedchain/llm/base.py index 35251719..8013ed2c 100644 --- a/embedchain/llm/base.py +++ b/embedchain/llm/base.py @@ -5,9 +5,9 @@ from langchain.memory import ConversationBufferMemory from langchain.schema import BaseMessage from embedchain.config import BaseLlmConfig -from embedchain.config.llm.base_llm_config import ( - DEFAULT_PROMPT, DEFAULT_PROMPT_WITH_HISTORY_TEMPLATE, - DOCS_SITE_PROMPT_TEMPLATE) +from embedchain.config.llm.base import (DEFAULT_PROMPT, + DEFAULT_PROMPT_WITH_HISTORY_TEMPLATE, + DOCS_SITE_PROMPT_TEMPLATE) from embedchain.helper.json_serializable import JSONSerializable @@ -174,7 +174,7 @@ class BaseLlm(JSONSerializable): :type input_query: str :param contexts: Embeddings retrieved from the database to be used as context. :type contexts: List[str] - :param config: The `LlmConfig` instance to use as configuration options. This is used for one method call. + :param config: The `BaseLlmConfig` instance to use as configuration options. This is used for one method call. To persistently use a config, declare it during app init., defaults to None :type config: Optional[BaseLlmConfig], optional :param dry_run: A dry run does everything except send the resulting prompt to @@ -230,7 +230,7 @@ class BaseLlm(JSONSerializable): :type input_query: str :param contexts: Embeddings retrieved from the database to be used as context. :type contexts: List[str] - :param config: The `LlmConfig` instance to use as configuration options. This is used for one method call. + :param config: The `BaseLlmConfig` instance to use as configuration options. This is used for one method call. To persistently use a config, declare it during app init., defaults to None :type config: Optional[BaseLlmConfig], optional :param dry_run: A dry run does everything except send the resulting prompt to diff --git a/embedchain/llm/gpt4all.py b/embedchain/llm/gpt4all.py index 91667fa7..2fa68eca 100644 --- a/embedchain/llm/gpt4all.py +++ b/embedchain/llm/gpt4all.py @@ -30,11 +30,11 @@ class GPT4ALLLlm(BaseLlm): def _get_answer(self, prompt: str, config: BaseLlmConfig) -> Union[str, Iterable]: if config.model and config.model != self.config.model: raise RuntimeError( - "OpenSourceApp does not support switching models at runtime. Please create a new app instance." + "GPT4ALLLlm does not support switching models at runtime. Please create a new app instance." ) if config.system_prompt: - raise ValueError("OpenSourceApp does not support `system_prompt`") + raise ValueError("GPT4ALLLlm does not support `system_prompt`") response = self.instance.generate( prompt=prompt, diff --git a/embedchain/llm/hugging_face_hub.py b/embedchain/llm/huggingface.py similarity index 76% rename from embedchain/llm/hugging_face_hub.py rename to embedchain/llm/huggingface.py index 8a03623c..358b1c2a 100644 --- a/embedchain/llm/hugging_face_hub.py +++ b/embedchain/llm/huggingface.py @@ -10,10 +10,10 @@ from embedchain.llm.base import BaseLlm @register_deserializable -class HuggingFaceHubLlm(BaseLlm): +class HuggingFaceLlm(BaseLlm): def __init__(self, config: Optional[BaseLlmConfig] = None): - if "HUGGINGFACEHUB_ACCESS_TOKEN" not in os.environ: - raise ValueError("Please set the HUGGINGFACEHUB_ACCESS_TOKEN environment variable.") + if "HUGGINGFACE_ACCESS_TOKEN" not in os.environ: + raise ValueError("Please set the HUGGINGFACE_ACCESS_TOKEN environment variable.") try: importlib.import_module("huggingface_hub") @@ -27,8 +27,8 @@ class HuggingFaceHubLlm(BaseLlm): def get_llm_model_answer(self, prompt): if self.config.system_prompt: - raise ValueError("HuggingFaceHubLlm does not support `system_prompt`") - return HuggingFaceHubLlm._get_answer(prompt=prompt, config=self.config) + raise ValueError("HuggingFaceLlm does not support `system_prompt`") + return HuggingFaceLlm._get_answer(prompt=prompt, config=self.config) @staticmethod def _get_answer(prompt: str, config: BaseLlmConfig) -> str: @@ -43,7 +43,7 @@ class HuggingFaceHubLlm(BaseLlm): raise ValueError("`top_p` must be > 0.0 and < 1.0") llm = HuggingFaceHub( - huggingfacehub_api_token=os.environ["HUGGINGFACEHUB_ACCESS_TOKEN"], + huggingfacehub_api_token=os.environ["HUGGINGFACE_ACCESS_TOKEN"], repo_id=config.model or "google/flan-t5-xxl", model_kwargs=model_kwargs, ) diff --git a/embedchain/llm/vertex_ai.py b/embedchain/llm/vertex_ai.py index af33a085..f2adc9ae 100644 --- a/embedchain/llm/vertex_ai.py +++ b/embedchain/llm/vertex_ai.py @@ -7,12 +7,12 @@ from embedchain.llm.base import BaseLlm @register_deserializable -class VertexAiLlm(BaseLlm): +class VertexAILlm(BaseLlm): def __init__(self, config: Optional[BaseLlmConfig] = None): super().__init__(config=config) def get_llm_model_answer(self, prompt): - return VertexAiLlm._get_answer(prompt=prompt, config=self.config) + return VertexAILlm._get_answer(prompt=prompt, config=self.config) @staticmethod def _get_answer(prompt: str, config: BaseLlmConfig) -> str: diff --git a/pyproject.toml b/pyproject.toml index d62bb2fe..cb71000e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,15 @@ [tool.poetry] name = "embedchain" -version = "0.0.69" -description = "embedchain is a framework to easily create LLM powered bots over any dataset" -authors = ["Taranjeet Singh"] +version = "0.0.70" +description = "Embedchain is a framework to easily create LLM powered apps over any dataset" +authors = ["Taranjeet Singh, Deshraj Yadav"] license = "Apache License" readme = "README.md" +exclude = [ + "db", + "configs", + "notebooks" +] [build-system] requires = ["setuptools", "wheel"] @@ -36,7 +41,7 @@ exclude = [ "build", "dist", "node_modules", - "venv", + "venv" ] line-length = 120 dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" diff --git a/tests/apps/test_apps.py b/tests/apps/test_apps.py index f046b560..c754982a 100644 --- a/tests/apps/test_apps.py +++ b/tests/apps/test_apps.py @@ -111,7 +111,7 @@ class TestAppFromConfig: return yaml.safe_load(file) def test_from_chroma_config(self): - yaml_path = "embedchain/yaml/chroma.yaml" + yaml_path = "configs/chroma.yaml" config_data = self.load_config_data(yaml_path) app = App.from_config(yaml_path) @@ -144,7 +144,7 @@ class TestAppFromConfig: assert app.embedder.config.deployment_name == embedder_config["deployment_name"] def test_from_opensource_config(self): - yaml_path = "embedchain/yaml/opensource.yaml" + yaml_path = "configs/opensource.yaml" config_data = self.load_config_data(yaml_path) app = App.from_config(yaml_path) diff --git a/tests/llm/test_chat.py b/tests/llm/test_chat.py index 83d0959d..2fcb7a11 100644 --- a/tests/llm/test_chat.py +++ b/tests/llm/test_chat.py @@ -85,11 +85,11 @@ class TestApp(unittest.TestCase): a where filter and 'get_llm_model_answer' returns an expected answer string. The 'chat' method is expected to call 'retrieve_from_database' with the where filter specified - in the LlmConfig and 'get_llm_model_answer' methods appropriately and return the right answer. + in the BaseLlmConfig and 'get_llm_model_answer' methods appropriately and return the right answer. Key assumptions tested: - 'retrieve_from_database' method is called exactly once with arguments: "Test query" and an instance of - LLmConfig. + BaseLlmConfig. - 'get_llm_model_answer' is called exactly once. The specific arguments are not checked in this test. - 'chat' method returns the value it received from 'get_llm_model_answer'. diff --git a/tests/llm/test_generate_prompt.py b/tests/llm/test_generate_prompt.py index c51c074d..0058032d 100644 --- a/tests/llm/test_generate_prompt.py +++ b/tests/llm/test_generate_prompt.py @@ -12,7 +12,7 @@ class TestGeneratePrompt(unittest.TestCase): def test_generate_prompt_with_template(self): """ Tests that the generate_prompt method correctly formats the prompt using - a custom template provided in the LlmConfig instance. + a custom template provided in the BaseLlmConfig instance. This test sets up a scenario with an input query and a list of contexts, and a custom template, and then calls generate_prompt. It checks that the @@ -58,7 +58,7 @@ class TestGeneratePrompt(unittest.TestCase): def test_generate_prompt_with_history(self): """ - Test the 'generate_prompt' method with LlmConfig containing a history attribute. + Test the 'generate_prompt' method with BaseLlmConfig containing a history attribute. """ config = BaseLlmConfig() config.template = Template("Context: $context | Query: $query | History: $history") diff --git a/tests/llm/test_hugging_face_hub.py b/tests/llm/test_huggingface.py similarity index 68% rename from tests/llm/test_hugging_face_hub.py rename to tests/llm/test_huggingface.py index d4d9d64d..cc1c5d9c 100644 --- a/tests/llm/test_hugging_face_hub.py +++ b/tests/llm/test_huggingface.py @@ -4,21 +4,21 @@ import unittest from unittest.mock import MagicMock, patch from embedchain.config import BaseLlmConfig -from embedchain.llm.hugging_face_hub import HuggingFaceHubLlm +from embedchain.llm.huggingface import HuggingFaceLlm -class TestHuggingFaceHubLlm(unittest.TestCase): +class TestHuggingFaceLlm(unittest.TestCase): def setUp(self): - os.environ["HUGGINGFACEHUB_ACCESS_TOKEN"] = "test_access_token" + os.environ["HUGGINGFACE_ACCESS_TOKEN"] = "test_access_token" self.config = BaseLlmConfig(model="google/flan-t5-xxl", max_tokens=50, temperature=0.7, top_p=0.8) def test_init_raises_value_error_without_api_key(self): - os.environ.pop("HUGGINGFACEHUB_ACCESS_TOKEN") + os.environ.pop("HUGGINGFACE_ACCESS_TOKEN") with self.assertRaises(ValueError): - HuggingFaceHubLlm() + HuggingFaceLlm() def test_get_llm_model_answer_raises_value_error_for_system_prompt(self): - llm = HuggingFaceHubLlm(self.config) + llm = HuggingFaceLlm(self.config) llm.config.system_prompt = "system_prompt" with self.assertRaises(ValueError): llm.get_llm_model_answer("prompt") @@ -26,7 +26,7 @@ class TestHuggingFaceHubLlm(unittest.TestCase): def test_top_p_value_within_range(self): config = BaseLlmConfig(top_p=1.0) with self.assertRaises(ValueError): - HuggingFaceHubLlm._get_answer("test_prompt", config) + HuggingFaceLlm._get_answer("test_prompt", config) def test_dependency_is_imported(self): importlib_installed = True @@ -36,27 +36,27 @@ class TestHuggingFaceHubLlm(unittest.TestCase): importlib_installed = False self.assertTrue(importlib_installed) - @patch("embedchain.llm.hugging_face_hub.HuggingFaceHubLlm._get_answer") + @patch("embedchain.llm.huggingface.HuggingFaceLlm._get_answer") def test_get_llm_model_answer(self, mock_get_answer): mock_get_answer.return_value = "Test answer" - llm = HuggingFaceHubLlm(self.config) + llm = HuggingFaceLlm(self.config) answer = llm.get_llm_model_answer("Test query") self.assertEqual(answer, "Test answer") mock_get_answer.assert_called_once() - @patch("embedchain.llm.hugging_face_hub.HuggingFaceHub") - def test_hugging_face_mock(self, mock_hugging_face_hub): + @patch("embedchain.llm.huggingface.HuggingFaceHub") + def test_hugging_face_mock(self, mock_huggingface): mock_llm_instance = MagicMock() mock_llm_instance.return_value = "Test answer" - mock_hugging_face_hub.return_value = mock_llm_instance + mock_huggingface.return_value = mock_llm_instance - llm = HuggingFaceHubLlm(self.config) + llm = HuggingFaceLlm(self.config) answer = llm.get_llm_model_answer("Test query") self.assertEqual(answer, "Test answer") - mock_hugging_face_hub.assert_called_once_with( + mock_huggingface.assert_called_once_with( huggingfacehub_api_token="test_access_token", repo_id="google/flan-t5-xxl", model_kwargs={"temperature": 0.7, "max_new_tokens": 50, "top_p": 0.8}, diff --git a/tests/llm/test_query.py b/tests/llm/test_query.py index 625ced41..e3b11afd 100644 --- a/tests/llm/test_query.py +++ b/tests/llm/test_query.py @@ -24,7 +24,7 @@ class TestApp(unittest.TestCase): Key assumptions tested: - 'retrieve_from_database' method is called exactly once with arguments: "Test query" and an instance of - LlmConfig. + BaseLlmConfig. - 'get_llm_model_answer' is called exactly once. The specific arguments are not checked in this test. - 'query' method returns the value it received from 'get_llm_model_answer'. @@ -82,7 +82,7 @@ class TestApp(unittest.TestCase): Key assumptions tested: - 'retrieve_from_database' method is called exactly once with arguments: "Test query" and an instance of - LlmConfig. + BaseLlmConfig. - 'get_llm_model_answer' is called exactly once. The specific arguments are not checked in this test. - 'query' method returns the value it received from 'get_llm_model_answer'. @@ -113,7 +113,7 @@ class TestApp(unittest.TestCase): Key assumptions tested: - 'retrieve_from_database' method is called exactly once with arguments: "Test query" and an instance of - LlmConfig. + BaseLlmConfig. - 'get_llm_model_answer' is called exactly once. The specific arguments are not checked in this test. - 'query' method returns the value it received from 'get_llm_model_answer'. diff --git a/tests/llm/test_vertex_ai.py b/tests/llm/test_vertex_ai.py index 9941523d..a9cfeeed 100644 --- a/tests/llm/test_vertex_ai.py +++ b/tests/llm/test_vertex_ai.py @@ -4,17 +4,17 @@ import pytest from langchain.schema import HumanMessage, SystemMessage from embedchain.config import BaseLlmConfig -from embedchain.llm.vertex_ai import VertexAiLlm +from embedchain.llm.vertex_ai import VertexAILlm @pytest.fixture def vertexai_llm(): config = BaseLlmConfig(temperature=0.6, model="vertexai_model", system_prompt="System Prompt") - return VertexAiLlm(config) + return VertexAILlm(config) def test_get_llm_model_answer(vertexai_llm): - with patch.object(VertexAiLlm, "_get_answer", return_value="Test Response") as mock_method: + with patch.object(VertexAILlm, "_get_answer", return_value="Test Response") as mock_method: prompt = "Test Prompt" response = vertexai_llm.get_llm_model_answer(prompt) assert response == "Test Response" diff --git a/tests/test_factory.py b/tests/test_factory.py index 5c6e81ed..1e3bcee5 100644 --- a/tests/test_factory.py +++ b/tests/test_factory.py @@ -34,12 +34,12 @@ class TestFactories: {"model": "sentence-transformers/all-mpnet-base-v2"}, embedchain.embedder.huggingface.HuggingFaceEmbedder, ), - ("vertexai", {"model": "textembedding-gecko"}, embedchain.embedder.vertexai.VertexAiEmbedder), + ("vertexai", {"model": "textembedding-gecko"}, embedchain.embedder.vertexai.VertexAIEmbedder), ("openai", {}, embedchain.embedder.openai.OpenAIEmbedder), ], ) def test_embedder_factory_create(self, mocker, provider_name, config_data, expected_class): - mocker.patch("embedchain.embedder.vertexai.VertexAiEmbedder", autospec=True) + mocker.patch("embedchain.embedder.vertexai.VertexAIEmbedder", autospec=True) embedder_instance = EmbedderFactory.create(provider_name, config_data) assert isinstance(embedder_instance, expected_class)