Support for Excel files (#1319)

embedchain/chunkers/excel_file.py

@@ -0,0 +1,22 @@
+from typing import Optional
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+from embedchain.chunkers.base_chunker import BaseChunker
+from embedchain.config.add_config import ChunkerConfig
+from embedchain.helpers.json_serializable import register_deserializable
+
+
+@register_deserializable
+class ExcelFileChunker(BaseChunker):
+    """Chunker for Excel file."""
+
+    def __init__(self, config: Optional[ChunkerConfig] = None):
+        if config is None:
+            config = ChunkerConfig(chunk_size=1000, chunk_overlap=0, length_function=len)
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=config.chunk_size,
+            chunk_overlap=config.chunk_overlap,
+            length_function=config.length_function,
+        )
+        super().__init__(text_splitter)

embedchain/data_formatter/data_formatter.py

@@ -80,6 +80,7 @@ class DataFormatter(JSONSerializable):
             DataType.SLACK: "embedchain.loaders.slack.SlackLoader",
             DataType.DROPBOX: "embedchain.loaders.dropbox.DropboxLoader",
             DataType.TEXT_FILE: "embedchain.loaders.text_file.TextFileLoader",
+            DataType.EXCEL_FILE: "embedchain.loaders.excel_file.ExcelFileLoader",
         }
 
         if data_type == DataType.CUSTOM or loader is not None:
@@ -127,6 +128,7 @@ class DataFormatter(JSONSerializable):
             DataType.SLACK: "embedchain.chunkers.common_chunker.CommonChunker",
             DataType.DROPBOX: "embedchain.chunkers.common_chunker.CommonChunker",
             DataType.TEXT_FILE: "embedchain.chunkers.common_chunker.CommonChunker",
+            DataType.EXCEL_FILE: "embedchain.chunkers.excel_file.ExcelFileChunker",
         }
 
         if chunker is not None:

embedchain/loaders/excel_file.py

@@ -0,0 +1,40 @@
+import hashlib
+import importlib.util
+
+try:
+    from langchain_community.document_loaders import UnstructuredExcelLoader
+except ImportError:
+    raise ImportError(
+        'Excel file requires extra dependencies. Install with `pip install --upgrade "embedchain[dataloaders]"`'
+    ) from None
+
+if importlib.util.find_spec("openpyxl") is None and importlib.util.find_spec("xlrd") is None:
+    raise ImportError("Excel file requires extra dependencies. Install with `pip install openpyxl xlrd`") from None
+
+from embedchain.helpers.json_serializable import register_deserializable
+from embedchain.loaders.base_loader import BaseLoader
+from embedchain.utils.misc import clean_string
+
+
+@register_deserializable
+class ExcelFileLoader(BaseLoader):
+    def load_data(self, excel_url):
+        """Load data from a Excel file."""
+        loader = UnstructuredExcelLoader(excel_url)
+        pages = loader.load_and_split()
+
+        data = []
+        for page in pages:
+            content = page.page_content
+            content = clean_string(content)
+
+            metadata = page.metadata
+            metadata["url"] = excel_url
+
+            data.append({"content": content, "meta_data": metadata})
+
+        doc_id = hashlib.sha256((content + excel_url).encode()).hexdigest()
+        return {
+            "doc_id": doc_id,
+            "data": data,
+        }

embedchain/models/data_type.py

@@ -40,6 +40,7 @@ class IndirectDataType(Enum):
     SLACK = "slack"
     DROPBOX = "dropbox"
     TEXT_FILE = "text_file"
+    EXCEL_FILE = "excel_file"
 
 
 class SpecialDataType(Enum):
@@ -79,3 +80,4 @@ class DataType(Enum):
     SLACK = IndirectDataType.SLACK.value
     DROPBOX = IndirectDataType.DROPBOX.value
     TEXT_FILE = IndirectDataType.TEXT_FILE.value
+    EXCEL_FILE = IndirectDataType.EXCEL_FILE.value