feat: csv loader (#470)

Co-authored-by: Taranjeet Singh <reachtotj@gmail.com>
This commit is contained in:
cachho
2023-09-05 10:18:03 +02:00
committed by GitHub
parent 344e7470f6
commit bd595f84e8
7 changed files with 172 additions and 0 deletions

View File

@@ -3,11 +3,13 @@ from embedchain.chunkers.docx_file import DocxFileChunker
from embedchain.chunkers.notion import NotionChunker
from embedchain.chunkers.pdf_file import PdfFileChunker
from embedchain.chunkers.qna_pair import QnaPairChunker
from embedchain.chunkers.table import TableChunker
from embedchain.chunkers.text import TextChunker
from embedchain.chunkers.web_page import WebPageChunker
from embedchain.chunkers.youtube_video import YoutubeVideoChunker
from embedchain.config import AddConfig
from embedchain.helper_classes.json_serializable import JSONSerializable
from embedchain.loaders.csv import CsvLoader
from embedchain.loaders.docs_site_loader import DocsSiteLoader
from embedchain.loaders.docx_file import DocxFileLoader
from embedchain.loaders.local_qna_pair import LocalQnaPairLoader
@@ -47,6 +49,7 @@ class DataFormatter(JSONSerializable):
DataType.DOCX: DocxFileLoader,
DataType.SITEMAP: SitemapLoader,
DataType.DOCS_SITE: DocsSiteLoader,
DataType.CSV: CsvLoader,
}
lazy_loaders = {DataType.NOTION}
if data_type in loaders:
@@ -81,6 +84,7 @@ class DataFormatter(JSONSerializable):
DataType.WEB_PAGE: WebPageChunker,
DataType.DOCS_SITE: DocsSiteChunker,
DataType.NOTION: NotionChunker,
DataType.CSV: TableChunker,
}
if data_type in chunker_classes:
chunker_class = chunker_classes[data_type]