Class TokenTextSplitter

Namespace

Mythosia.AI.Rag.Splitters

Assembly

Mythosia.AI.Rag.dll

public class TokenTextSplitter : ITextSplitter

Inheritance

object

TokenTextSplitter

Implements

ITextSplitter

Inherited Members

object.GetType()

object.MemberwiseClone()

object.ToString()

object.Equals(object)

object.Equals(object, object)

object.ReferenceEquals(object, object)

object.GetHashCode()

Constructors

TokenTextSplitter()

public TokenTextSplitter()

TokenTextSplitter(int, int)

public TokenTextSplitter(int maxTokensPerChunk, int tokenOverlap = 50)

Parameters

maxTokensPerChunk int

tokenOverlap int

Properties

MaxTokensPerChunk

public int MaxTokensPerChunk { get; set; }

Property Value

int

TokenOverlap

public int TokenOverlap { get; set; }

Property Value

int

TokenSeparators

public char[] TokenSeparators { get; set; }

Property Value

char[]

Methods

Split(RagDocument)

public IReadOnlyList<RagChunk> Split(RagDocument document)

Parameters

document RagDocument

Returns

IReadOnlyList<RagChunk>