feat: Add embedding manager (#570)

tests/chunkers/test_text.py

@@ -69,9 +69,12 @@ class MockLoader:
         Mock loader that returns a list of data dictionaries.
         Adjust this method to return different data for testing.
         """
-        return [
-            {
-                "content": src,
-                "meta_data": {"url": "none"},
-            }
-        ]
+        return {
+            "doc_id": "123",
+            "data": [
+                {
+                    "content": src,
+                    "meta_data": {"url": "none"},
+                }
+            ]
+        }

tests/loaders/test_csv.py

@@ -29,18 +29,19 @@ def test_load_data(delimiter):
         # Loading CSV using CsvLoader
         loader = CsvLoader()
         result = loader.load_data(filename)
+        data = result["data"]
 
         # Assertions
-        assert len(result) == 3
-        assert result[0]["content"] == "Name: Alice, Age: 28, Occupation: Engineer"
-        assert result[0]["meta_data"]["url"] == filename
-        assert result[0]["meta_data"]["row"] == 1
-        assert result[1]["content"] == "Name: Bob, Age: 35, Occupation: Doctor"
-        assert result[1]["meta_data"]["url"] == filename
-        assert result[1]["meta_data"]["row"] == 2
-        assert result[2]["content"] == "Name: Charlie, Age: 22, Occupation: Student"
-        assert result[2]["meta_data"]["url"] == filename
-        assert result[2]["meta_data"]["row"] == 3
+        assert len(data) == 3
+        assert data[0]["content"] == "Name: Alice, Age: 28, Occupation: Engineer"
+        assert data[0]["meta_data"]["url"] == filename
+        assert data[0]["meta_data"]["row"] == 1
+        assert data[1]["content"] == "Name: Bob, Age: 35, Occupation: Doctor"
+        assert data[1]["meta_data"]["url"] == filename
+        assert data[1]["meta_data"]["row"] == 2
+        assert data[2]["content"] == "Name: Charlie, Age: 22, Occupation: Student"
+        assert data[2]["meta_data"]["url"] == filename
+        assert data[2]["meta_data"]["row"] == 3
 
         # Cleaning up the temporary file
         os.unlink(filename)
@@ -67,18 +68,19 @@ def test_load_data_with_file_uri(delimiter):
         # Loading CSV using CsvLoader
         loader = CsvLoader()
         result = loader.load_data(filename)
+        data = result["data"]
 
         # Assertions
-        assert len(result) == 3
-        assert result[0]["content"] == "Name: Alice, Age: 28, Occupation: Engineer"
-        assert result[0]["meta_data"]["url"] == filename
-        assert result[0]["meta_data"]["row"] == 1
-        assert result[1]["content"] == "Name: Bob, Age: 35, Occupation: Doctor"
-        assert result[1]["meta_data"]["url"] == filename
-        assert result[1]["meta_data"]["row"] == 2
-        assert result[2]["content"] == "Name: Charlie, Age: 22, Occupation: Student"
-        assert result[2]["meta_data"]["url"] == filename
-        assert result[2]["meta_data"]["row"] == 3
+        assert len(data) == 3
+        assert data[0]["content"] == "Name: Alice, Age: 28, Occupation: Engineer"
+        assert data[0]["meta_data"]["url"] == filename
+        assert data[0]["meta_data"]["row"] == 1
+        assert data[1]["content"] == "Name: Bob, Age: 35, Occupation: Doctor"
+        assert data[1]["meta_data"]["url"] == filename
+        assert data[1]["meta_data"]["row"] == 2
+        assert data[2]["content"] == "Name: Charlie, Age: 22, Occupation: Student"
+        assert data[2]["meta_data"]["url"] == filename
+        assert data[2]["meta_data"]["row"] == 3
 
         # Cleaning up the temporary file
         os.unlink(tmpfile.name)