klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add batch_size in config for VectorDB (#1448)

Browse Source
This commit is contained in:
Dev Khant
2024-06-28 03:15:58 +05:30
committed by GitHub
parent edaeb78ccf
commit 0a78198bb5
10 changed files with 28 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
embedchain/vectordb/opensearch.py
View File

@@ -28,8 +28,6 @@ class OpenSearchDB(BaseVectorDB):
OpenSearch as vector database
"""
BATCH_SIZE = 100
def __init__(self, config: OpenSearchDBConfig):
"""OpenSearch as vector database.
@@ -120,8 +118,10 @@ class OpenSearchDB(BaseVectorDB):
"""Adds documents to the opensearch index"""
embeddings = self.embedder.embedding_fn(documents)
for batch_start in tqdm(range(0, len(documents), self.BATCH_SIZE), desc="Inserting batches in opensearch"):
batch_end = batch_start + self.BATCH_SIZE
for batch_start in tqdm(
range(0, len(documents), self.config.batch_size), desc="Inserting batches in opensearch"
):
batch_end = batch_start + self.config.batch_size
batch_documents = documents[batch_start:batch_end]
batch_embeddings = embeddings[batch_start:batch_end]