unclecode/crawl4ai
1
0
Fork 0
mirror of https://github.com/unclecode/crawl4ai.git synced 2026-06-11 00:08:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1913 from unclecode/fix/nlp-sentence-chunking-1909

Browse Source 
fix(chunking): preserve sentence order in NlpSentenceChunking
This commit is contained in:
Nasrin
2026-04-16 10:24:59 +02:00
committed by GitHub
parent 7bfc547bce c837c0d9cb
commit 4e86399bfa
1 changed files with 1 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crawl4ai/chunking_strategy.py
View File

@@ -71,7 +71,6 @@ class NlpSentenceChunking(ChunkingStrategy):
"""
Initialize the NlpSentenceChunking object.
"""
from crawl4ai.le.legacy.model_loader import load_nltk_punkt
load_nltk_punkt()
def chunk(self, text: str) -> list:
@@ -86,7 +85,7 @@ class NlpSentenceChunking(ChunkingStrategy):
sentences = sent_tokenize(text)
sens = [sent.strip() for sent in sentences]
return list(set(sens))
return sens
# Topic-based segmentation using TextTiling