klas/t6_mem0
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Features] Add Github and Youtube Channel loaders (#957)

Browse Source 
Co-authored-by: Deven Patel <deven298@yahoo.com>
Co-authored-by: Deshraj Yadav <deshrajdry@gmail.com>
This commit is contained in:
Deven Patel
2023-11-15 19:17:42 -08:00
committed by GitHub
parent 3fa7db8420
commit 07fb6bee54
9 changed files with 303 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
embedchain/data_formatter/data_formatter.py
View File

@@ -64,6 +64,8 @@ class DataFormatter(JSONSerializable):
DataType.GMAIL: "embedchain.loaders.gmail.GmailLoader",
DataType.NOTION: "embedchain.loaders.notion.NotionLoader",
DataType.SUBSTACK: "embedchain.loaders.substack.SubstackLoader",
DataType.GITHUB: "embedchain.loaders.github.GithubLoader",
DataType.YOUTUBE_CHANNEL: "embedchain.loaders.youtube_channel.YoutubeChannelLoader",
}
custom_loaders = set(
@@ -114,6 +116,8 @@ class DataFormatter(JSONSerializable):
DataType.SLACK: "embedchain.chunkers.slack.SlackChunker",
DataType.DISCOURSE: "embedchain.chunkers.discourse.DiscourseChunker",
DataType.SUBSTACK: "embedchain.chunkers.substack.SubstackChunker",
DataType.GITHUB: "embedchain.chunkers.base_chunker.BaseChunker",
DataType.YOUTUBE_CHANNEL: "embedchain.chunkers.base_chunker.BaseChunker",
}
if data_type in chunker_classes: