Load local text files of any kind - code, txts, json etc (#1076)

embedchain/loaders/directory_loader.py

@@ -7,7 +7,7 @@ from embedchain.config import AddConfig
 from embedchain.data_formatter.data_formatter import DataFormatter
 from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
-from embedchain.loaders.local_text import LocalTextLoader
+from embedchain.loaders.text_file import TextFileLoader
 from embedchain.utils import detect_datatype
 
 
@@ -58,4 +58,4 @@ class DirectoryLoader(BaseLoader):
             )
         except Exception as e:
             self.errors.append(f"Error processing {file_path}: {e}")
-            return LocalTextLoader()
+            return TextFileLoader()

embedchain/loaders/text_file.py

@@ -0,0 +1,30 @@
+import hashlib
+import os
+
+from embedchain.helpers.json_serializable import register_deserializable
+from embedchain.loaders.base_loader import BaseLoader
+
+
+@register_deserializable
+class TextFileLoader(BaseLoader):
+    def load_data(self, url: str):
+        """Load data from a text file located at a local path."""
+        if not os.path.exists(url):
+            raise FileNotFoundError(f"The file at {url} does not exist.")
+
+        with open(url, "r", encoding="utf-8") as file:
+            content = file.read()
+
+        doc_id = hashlib.sha256((content + url).encode()).hexdigest()
+
+        meta_data = {"url": url, "file_size": os.path.getsize(url), "file_type": url.split(".")[-1]}
+
+        return {
+            "doc_id": doc_id,
+            "data": [
+                {
+                    "content": content,
+                    "meta_data": meta_data,
+                }
+            ],
+        }