perf: allow zero chunk overlap in recursive chunker (#4258)
* Allow zero chunk overlap * Validate recursive chunking bounds
This commit is contained in:
@@ -149,8 +149,16 @@ class RecursiveCharacterChunker(BaseChunker):
|
||||
分割后的文本块列表
|
||||
|
||||
"""
|
||||
chunk_size = chunk_size or self.chunk_size
|
||||
overlap = overlap or self.chunk_overlap
|
||||
if chunk_size is None:
|
||||
chunk_size = self.chunk_size
|
||||
if overlap is None:
|
||||
overlap = self.chunk_overlap
|
||||
if chunk_size <= 0:
|
||||
raise ValueError("chunk_size must be greater than 0")
|
||||
if overlap < 0:
|
||||
raise ValueError("chunk_overlap must be non-negative")
|
||||
if overlap >= chunk_size:
|
||||
raise ValueError("chunk_overlap must be less than chunk_size")
|
||||
result = []
|
||||
for i in range(0, len(text), chunk_size - overlap):
|
||||
end = min(i + chunk_size, len(text))
|
||||
|
||||
Reference in New Issue
Block a user