klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Improvements] Improve the default prompt and data loader util functions (#1272)

Browse Source
This commit is contained in:
Deshraj Yadav
2024-02-18 14:06:32 -08:00
committed by GitHub
parent 9a11683003
commit 6c12bc9044
19 changed files with 79 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
embedchain/config/llm/base.py
View File

@@ -7,40 +7,60 @@ from embedchain.config.base_config import BaseConfig
from embedchain.helpers.json_serializable import register_deserializable
DEFAULT_PROMPT = """
Use the following pieces of context to answer the query at the end.
If you don't know the answer, just say that you don't know, don't try to make up an answer.
You are a Q&A expert system. Your responses must always be rooted in the context provided for each query. Here are some guidelines to follow:
$context
1. Refrain from explicitly mentioning the context provided in your response.
2. The context should silently guide your answers without being directly acknowledged.
3. Do not use phrases such as 'According to the context provided', 'Based on the context, ...' etc.
Query: $query
Context information:
----------------------
$context
----------------------
Helpful Answer:
Query: $query
Answer:
""" # noqa:E501
DEFAULT_PROMPT_WITH_HISTORY = """
Use the following pieces of context to answer the query at the end.
If you don't know the answer, just say that you don't know, don't try to make up an answer.
I will provide you with our conversation history.
You are a Q&A expert system. Your responses must always be rooted in the context provided for each query. You are also provided with the conversation history with the user. Make sure to use relevant context from conversation history as needed.
$context
Here are some guidelines to follow:
History:
$history
1. Refrain from explicitly mentioning the context provided in your response.
2. The context should silently guide your answers without being directly acknowledged.
3. Do not use phrases such as 'According to the context provided', 'Based on the context, ...' etc.
Query: $query
Context information:
----------------------
$context
----------------------
Helpful Answer:
Conversation history:
----------------------
$history
----------------------
Query: $query
Answer:
""" # noqa:E501
DOCS_SITE_DEFAULT_PROMPT = """
Use the following pieces of context to answer the query at the end.
If you don't know the answer, just say that you don't know, don't try to make up an answer. Wherever possible, give complete code snippet. Dont make up any code snippet on your own.
You are an expert AI assistant for developer support product. Your responses must always be rooted in the context provided for each query. Wherever possible, give complete code snippet. Dont make up any code snippet on your own.
$context
Here are some guidelines to follow:
Query: $query
1. Refrain from explicitly mentioning the context provided in your response.
2. The context should silently guide your answers without being directly acknowledged.
3. Do not use phrases such as 'According to the context provided', 'Based on the context, ...' etc.
Helpful Answer:
Context information:
----------------------
$context
----------------------
Query: $query
Answer:
""" # noqa:E501
DEFAULT_PROMPT_TEMPLATE = Template(DEFAULT_PROMPT)

4
embedchain/loaders/discord.py
View File

@@ -131,7 +131,7 @@ class DiscordLoader(BaseLoader):
client = DiscordClient(intents=intents)
client.run(self.token)
meta_data = {
metadata = {
"url": channel_id,
}
@@ -144,7 +144,7 @@ class DiscordLoader(BaseLoader):
"data": [
{
"content": messages,
"meta_data": meta_data,
"meta_data": metadata,
}
],
}

4
embedchain/loaders/discourse.py
View File

@@ -39,7 +39,7 @@ class DiscourseLoader(BaseLoader):
return
response_data = response.json()
post_contents = clean_string(response_data.get("raw"))
meta_data = {
metadata = {
"url": post_url,
"created_at": response_data.get("created_at", ""),
"username": response_data.get("username", ""),
@@ -48,7 +48,7 @@ class DiscourseLoader(BaseLoader):
}
data = {
"content": post_contents,
"meta_data": meta_data,
"meta_data": metadata,
}
return data

6
embedchain/loaders/docx_file.py
View File

@@ -18,9 +18,9 @@ class DocxFileLoader(BaseLoader):
output = []
data = loader.load()
content = data[0].page_content
meta_data = data[0].metadata
meta_data["url"] = "local"
output.append({"content": content, "meta_data": meta_data})
metadata = data[0].metadata
metadata["url"] = "local"
output.append({"content": content, "meta_data": metadata})
doc_id = hashlib.sha256((content + url).encode()).hexdigest()
return {
"doc_id": doc_id,

4
embedchain/loaders/local_qna_pair.py
View File

@@ -11,14 +11,14 @@ class LocalQnaPairLoader(BaseLoader):
question, answer = content
content = f"Q: {question}\nA: {answer}"
url = "local"
meta_data = {"url": url, "question": question}
metadata = {"url": url, "question": question}
doc_id = hashlib.sha256((content + url).encode()).hexdigest()
return {
"doc_id": doc_id,
"data": [
{
"content": content,
"meta_data": meta_data,
"meta_data": metadata,
}
],
}

4
embedchain/loaders/local_text.py
View File

@@ -9,7 +9,7 @@ class LocalTextLoader(BaseLoader):
def load_data(self, content):
"""Load data from a local text file."""
url = "local"
meta_data = {
metadata = {
"url": url,
}
doc_id = hashlib.sha256((content + url).encode()).hexdigest()
@@ -18,7 +18,7 @@ class LocalTextLoader(BaseLoader):
"data": [
{
"content": content,
"meta_data": meta_data,
"meta_data": metadata,
}
],
}

4
embedchain/loaders/mdx.py
View File

@@ -10,7 +10,7 @@ class MdxLoader(BaseLoader):
"""Load data from a mdx file."""
with open(url, "r", encoding="utf-8") as infile:
content = infile.read()
meta_data = {
metadata = {
"url": url,
}
doc_id = hashlib.sha256((content + url).encode()).hexdigest()
@@ -19,7 +19,7 @@ class MdxLoader(BaseLoader):
"data": [
{
"content": content,
"meta_data": meta_data,
"meta_data": metadata,
}
],
}

4
embedchain/loaders/openapi.py
View File

@@ -35,8 +35,8 @@ class OpenAPILoader(BaseLoader):
yaml_data = yaml.load(file, Loader=yaml.SafeLoader)
for i, (key, value) in enumerate(yaml_data.items()):
string_data = f"{key}: {value}"
meta_data = {"url": file_path, "row": i + 1}
data.append({"content": string_data, "meta_data": meta_data})
metadata = {"url": file_path, "row": i + 1}
data.append({"content": string_data, "meta_data": metadata})
data_content.append(string_data)
doc_id = hashlib.sha256((content + ", ".join(data_content)).encode()).hexdigest()
return {"doc_id": doc_id, "data": data}

6
embedchain/loaders/pdf_file.py
View File

@@ -27,12 +27,12 @@ class PdfFileLoader(BaseLoader):
for page in pages:
content = page.page_content
content = clean_string(content)
meta_data = page.metadata
meta_data["url"] = url
metadata = page.metadata
metadata["url"] = url
data.append(
{
"content": content,
"meta_data": meta_data,
"meta_data": metadata,
}
)
all_content.append(content)

6
embedchain/loaders/rss_feed.py
View File

@@ -41,12 +41,12 @@ class RSSFeedLoader(BaseLoader):
data = loader.load()
for entry in data:
meta_data = RSSFeedLoader.serialize_metadata(entry.metadata)
meta_data.update({"url": url})
metadata = RSSFeedLoader.serialize_metadata(entry.metadata)
metadata.update({"url": url})
output.append(
{
"content": entry.page_content,
"meta_data": meta_data,
"meta_data": metadata,
}
)

8
embedchain/loaders/slack.py
View File

@@ -88,16 +88,16 @@ class SlackLoader(BaseLoader):
content = clean_string(text)
message_meta_data_keys = ["iid", "team", "ts", "type", "user", "username"]
meta_data = {}
metadata = {}
for key in message.keys():
if key in message_meta_data_keys:
meta_data[key] = message.get(key)
meta_data.update({"url": url})
metadata[key] = message.get(key)
metadata.update({"url": url})
data.append(
{
"content": content,
"meta_data": meta_data,
"meta_data": metadata,
}
)
data_content.append(content)

4
embedchain/loaders/text_file.py
View File

@@ -17,14 +17,14 @@ class TextFileLoader(BaseLoader):
doc_id = hashlib.sha256((content + url).encode()).hexdigest()
meta_data = {"url": url, "file_size": os.path.getsize(url), "file_type": url.split(".")[-1]}
metadata = {"url": url, "file_size": os.path.getsize(url), "file_type": url.split(".")[-1]}
return {
"doc_id": doc_id,
"data": [
{
"content": content,
"meta_data": meta_data,
"meta_data": metadata,
}
],
}

6
embedchain/loaders/unstructured_file.py
View File

@@ -26,12 +26,12 @@ class UnstructuredLoader(BaseLoader):
for page in pages:
content = page.page_content
content = clean_string(content)
meta_data = page.metadata
meta_data["url"] = url
metadata = page.metadata
metadata["url"] = url
data.append(
{
"content": content,
"meta_data": meta_data,
"meta_data": metadata,
}
)
all_content.append(content)

4
embedchain/loaders/web_page.py
View File

@@ -30,7 +30,7 @@ class WebPageLoader(BaseLoader):
data = response.content
content = self._get_clean_content(data, url)
meta_data = {"url": url}
metadata = {"url": url}
doc_id = hashlib.sha256((content + url).encode()).hexdigest()
return {
@@ -38,7 +38,7 @@ class WebPageLoader(BaseLoader):
"data": [
{
"content": content,
"meta_data": meta_data,
"meta_data": metadata,
}
],
}

8
embedchain/loaders/xml.py
View File

@@ -19,10 +19,10 @@ class XmlLoader(BaseLoader):
data = loader.load()
content = data[0].page_content
content = clean_string(content)
meta_data = data[0].metadata
meta_data["url"] = meta_data["source"]
del meta_data["source"]
output = [{"content": content, "meta_data": meta_data}]
metadata = data[0].metadata
metadata["url"] = metadata["source"]
del metadata["source"]
output = [{"content": content, "meta_data": metadata}]
doc_id = hashlib.sha256((content + xml_url).encode()).hexdigest()
return {
"doc_id": doc_id,

6
embedchain/loaders/youtube_video.py
View File

@@ -22,13 +22,13 @@ class YoutubeVideoLoader(BaseLoader):
raise ValueError(f"No data found for url: {url}")
content = doc[0].page_content
content = clean_string(content)
meta_data = doc[0].metadata
meta_data["url"] = url
metadata = doc[0].metadata
metadata["url"] = url
output.append(
{
"content": content,
"meta_data": meta_data,
"meta_data": metadata,
}
)
doc_id = hashlib.sha256((content + url).encode()).hexdigest()

3
embedchain/utils/misc.py
View File

@@ -79,9 +79,6 @@ def clean_string(text):
cleaned_text (str): The cleaned text after all the cleaning operations
have been performed.
"""
# Replacement of newline characters:
text = text.replace("\n", " ")
# Stripping and reducing multiple spaces to single:
cleaned_text = re.sub(r"\s+", " ", text.strip())