klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feature: Add support for loading docs website (#293)

Browse Source
This commit is contained in:
Deshraj Yadav
2023-07-16 22:22:52 -07:00
committed by GitHub
parent d5e40e1853
commit a548863a09
10 changed files with 173 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
embedchain/data_formatter/data_formatter.py
View File

@@ -1,4 +1,4 @@
from embedchain.chunkers.code_docs_page import CodeDocsPageChunker
from embedchain.chunkers.docs_site import DocsSiteChunker
from embedchain.chunkers.docx_file import DocxFileChunker
from embedchain.chunkers.pdf_file import PdfFileChunker
from embedchain.chunkers.qna_pair import QnaPairChunker
@@ -6,7 +6,7 @@ from embedchain.chunkers.text import TextChunker
from embedchain.chunkers.web_page import WebPageChunker
from embedchain.chunkers.youtube_video import YoutubeVideoChunker
from embedchain.config import AddConfig
from embedchain.loaders.code_docs_page import CodeDocsPageLoader
from embedchain.loaders.docs_site_loader import DocsSiteLoader
from embedchain.loaders.docx_file import DocxFileLoader
from embedchain.loaders.local_qna_pair import LocalQnaPairLoader
from embedchain.loaders.local_text import LocalTextLoader
@@ -43,7 +43,7 @@ class DataFormatter:
"text": LocalTextLoader(),
"docx": DocxFileLoader(),
"sitemap": SitemapLoader(),
"code_docs_page": CodeDocsPageLoader(),
"docs_site": DocsSiteLoader(),
}
if data_type in loaders:
return loaders[data_type]
@@ -66,7 +66,7 @@ class DataFormatter:
"text": TextChunker(config),
"docx": DocxFileChunker(config),
"sitemap": WebPageChunker(config),
"code_docs_page": CodeDocsPageChunker(config),
"docs_site": DocsSiteChunker(config),
}
if data_type in chunkers:
return chunkers[data_type]