feat: Add embedding manager (#570)

2023-09-11 23:43:53 -07:00
parent ba208f5b48
commit 2bd6881361
16 changed files with 311 additions and 73 deletions
--- a/embedchain/loaders/web_page.py
+++ b/embedchain/loaders/web_page.py
@@ -1,3 +1,4 @@
+import hashlib
 import logging

 import requests
@@ -63,10 +64,14 @@ class WebPageLoader(BaseLoader):
        meta_data = {
            "url": url,
        }
-
-        return [
-            {
-                "content": content,
-                "meta_data": meta_data,
-            }
-        ]
+        content = content
+        doc_id = hashlib.sha256((content + url).encode()).hexdigest()
+        return  {
+            "doc_id": doc_id,
+            "data": [
+                {
+                    "content": content,
+                    "meta_data": meta_data,
+                }
+            ],
+        }