Text splitter bugfixes

src/Cli/test/LangChain.Cli.IntegrationTests/LangChain.Cli.IntegrationTests.csproj

@@ -7,6 +7,10 @@
         <Nullable>enable</Nullable>
     </PropertyGroup>
 
+    <ItemGroup>
+      <PackageReference Include="FluentAssertions" />
+    </ItemGroup>
+
     <ItemGroup>
       <ProjectReference Include="..\..\src\LangChain.Cli.csproj" />
     </ItemGroup>

src/Core/test/UnitTests/LangChain.Core.UnitTests.csproj

@@ -4,6 +4,10 @@
         <TargetFramework>net9.0</TargetFramework>
     </PropertyGroup>
 
+    <ItemGroup>
+      <PackageReference Include="FluentAssertions" />
+    </ItemGroup>
+
     <ItemGroup>
       <ProjectReference Include="..\..\src\LangChain.Core.csproj" />
     </ItemGroup>

src/Directory.Packages.props

@@ -9,6 +9,7 @@
     <PackageVersion Include="DocumentFormat.OpenXml" Version="3.2.0" />
     <PackageVersion Include="DotNet.ReproducibleBuilds" Version="1.2.25" />
     <PackageVersion Include="AwesomeAssertions" Version="9.0.0" />
+    <PackageVersion Include="FluentAssertions" Version="8.5.0" />
     <PackageVersion Include="GitHubActionsTestLogger" Version="2.4.1" />
     <PackageVersion Include="H.Resources.Generator" Version="1.8.0">
       <PrivateAssets>all</PrivateAssets>

src/DocumentLoaders/IntegrationTests/LangChain.DocumentLoaders.IntegrationTests.csproj

@@ -6,6 +6,10 @@
         <NoWarn>$(NoWarn)</NoWarn>
     </PropertyGroup>
 
+    <ItemGroup>
+      <PackageReference Include="FluentAssertions" />
+    </ItemGroup>
+
     <ItemGroup>
       <ProjectReference Include="..\Pdf\src\LangChain.DocumentLoaders.Pdf.csproj" />
       <ProjectReference Include="..\WebBase\src\LangChain.DocumentLoaders.Html.csproj" />

src/Meta/test/LangChain.IntegrationTests.csproj

@@ -6,6 +6,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="FluentAssertions" />
         <PackageReference Include="LangChain.Databases.OpenSearch" />
         <PackageReference Include="LangChain.Databases.Sqlite" />
         <PackageReference Include="LangChain.Providers.Amazon.Bedrock" />

src/Splitters/Abstractions/src/Text/CharacterTextSplitter.cs

@@ -15,6 +15,8 @@ public override IReadOnlyList<string> SplitText(string text)
     {
         text = text ?? throw new ArgumentNullException(nameof(text));
 
+        text = text.Replace("\r", ""); // some people are using windows
+
         List<string> splits;
         if (separator != null)
         {

src/Splitters/Abstractions/src/Text/MarkdownHeaderTextSplitter.cs

@@ -76,7 +76,7 @@ public override IReadOnlyList<string> SplitText(string text)
                 {
                     var existingHeader = currentHeader.Split('|');
 
-                    string prevHeader = string.Join("|", existingHeader.Take(existingHeader.Length - 1));
+                    string prevHeader = string.Join("|", existingHeader.Take(existingHeader.Length - (1 - hLen + currentHeaderLen)));
                     currentHeader = prevHeader + "|" + strippedLine.TrimStart('#').Trim();
                     currentHeaderLen = hLen;
                     continue;

src/Splitters/Abstractions/src/Text/RecursiveCharacterTextSplitter.cs

@@ -19,6 +19,8 @@ public override IReadOnlyList<string> SplitText(string text)
     {
         text = text ?? throw new ArgumentNullException(nameof(text));
 
+        text = text.Replace("\r", ""); // some people are using windows
+
         List<string> finalChunks = new List<string>();
         string separator = _separators[_separators.Count - 1];
 

src/Splitters/Abstractions/test/LangChain.Splitters.Abstractions.Tests.csproj

@@ -4,6 +4,10 @@
         <TargetFramework>net9.0</TargetFramework>
     </PropertyGroup>
 
+    <ItemGroup>
+      <PackageReference Include="FluentAssertions" />
+    </ItemGroup>
+
     <ItemGroup>
       <ProjectReference Include="..\..\..\Core\src\LangChain.Core.csproj" />
       <ProjectReference Include="..\src\LangChain.Splitters.Abstractions.csproj" />

src/Splitters/Abstractions/test/Resources/markdown_test_material.md

@@ -0,0 +1,71 @@
+# Header A
+
+Text A
+
+## Header A.A
+
+Text A.A
+
+## Header A.B
+
+Text A.B
+
+### Header A.B.A
+
+Text A.B.A
+
+### Header A.B.B
+
+Text A.B.B
+
+### Header A.B.C
+
+Text A.B.C
+
+## Header A.C
+
+Text A.C
+
+### Header A.C.A
+
+Text A.C.A
+
+### Header A.C.B
+
+Text A.C.B
+
+# Header B
+
+Text B
+
+## Header B.A
+
+Text B.A
+
+## Header B.B
+
+Text B.B
+
+### Header B.B.A
+
+Text B.B.A
+
+### Header B.B.B
+
+Text B.B.B
+
+## Header B.C
+
+Text B.C
+
+### Header B.C.A
+
+Text B.C.A
+
+### Header B.C.B
+
+Text B.C.B
+
+### Header B.C.C
+
+Text B.C.C

src/Splitters/Abstractions/test/Tests.MarkdownHeader.cs

@@ -61,4 +61,69 @@ Hi this is Joe
         res[0].Should().Be("Hi this is Jim\nHi this is Joe");
         res[1].Should().Be("Hi this is Molly");
     }
+
+    [Test]
+    public void TestMarkdown4()
+    {
+        var md = H.Resources.markdown_test_material_md.AsString();
+
+        var splitter = new MarkdownHeaderTextSplitter();
+        var res = splitter.SplitText(md);
+
+        res.Count.Should().Be(18);
+
+        res[0].Split("\n")[0].Should().Be("Header A");
+        res[0].Split("\n")[1].Should().Be("Text A");
+
+        res[1].Split("\n")[0].Should().Be("Header A: Header A.A");
+        res[1].Split("\n")[1].Should().Be("Text A.A");
+
+        res[2].Split("\n")[0].Should().Be("Header A: Header A.B");
+        res[2].Split("\n")[1].Should().Be("Text A.B");
+
+        res[3].Split("\n")[0].Should().Be("Header A: Header A.B: Header A.B.A");
+        res[3].Split("\n")[1].Should().Be("Text A.B.A");
+
+        res[4].Split("\n")[0].Should().Be("Header A: Header A.B: Header A.B.B");
+        res[4].Split("\n")[1].Should().Be("Text A.B.B");
+
+        res[5].Split("\n")[0].Should().Be("Header A: Header A.B: Header A.B.C");
+        res[5].Split("\n")[1].Should().Be("Text A.B.C");
+
+        res[6].Split("\n")[0].Should().Be("Header A: Header A.C");
+        res[6].Split("\n")[1].Should().Be("Text A.C");
+
+        res[7].Split("\n")[0].Should().Be("Header A: Header A.C: Header A.C.A");
+        res[7].Split("\n")[1].Should().Be("Text A.C.A");
+
+        res[8].Split("\n")[0].Should().Be("Header A: Header A.C: Header A.C.B");
+        res[8].Split("\n")[1].Should().Be("Text A.C.B");
+
+        res[9].Split("\n")[0].Should().Be("Header B");
+        res[9].Split("\n")[1].Should().Be("Text B");
+
+        res[10].Split("\n")[0].Should().Be("Header B: Header B.A");
+        res[10].Split("\n")[1].Should().Be("Text B.A");
+
+        res[11].Split("\n")[0].Should().Be("Header B: Header B.B");
+        res[11].Split("\n")[1].Should().Be("Text B.B");
+
+        res[12].Split("\n")[0].Should().Be("Header B: Header B.B: Header B.B.A");
+        res[12].Split("\n")[1].Should().Be("Text B.B.A");
+
+        res[13].Split("\n")[0].Should().Be("Header B: Header B.B: Header B.B.B");
+        res[13].Split("\n")[1].Should().Be("Text B.B.B");
+
+        res[14].Split("\n")[0].Should().Be("Header B: Header B.C");
+        res[14].Split("\n")[1].Should().Be("Text B.C");
+
+        res[15].Split("\n")[0].Should().Be("Header B: Header B.C: Header B.C.A");
+        res[15].Split("\n")[1].Should().Be("Text B.C.A");
+
+        res[16].Split("\n")[0].Should().Be("Header B: Header B.C: Header B.C.B");
+        res[16].Split("\n")[1].Should().Be("Text B.C.B");
+
+        res[17].Split("\n")[0].Should().Be("Header B: Header B.C: Header B.C.C");
+        res[17].Split("\n")[1].Should().Be("Text B.C.C");
+    }
 }

src/Splitters/CSharp/test/LangChain.Splitters.CSharp.Tests.csproj

@@ -4,6 +4,10 @@
         <TargetFramework>net9.0</TargetFramework>
     </PropertyGroup>
 
+    <ItemGroup>
+      <PackageReference Include="FluentAssertions" />
+    </ItemGroup>
+
     <ItemGroup>
         <ProjectReference Include="..\src\LangChain.Splitters.CSharp.csproj" />
     </ItemGroup>

src/Utilities/Postgres/test/LangChain.Utilities.Postgres.IntegrationTests.csproj

@@ -6,6 +6,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="FluentAssertions" />
         <PackageReference Include="LangChain.Providers.OpenAI" />
         <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
         <PackageReference Include="Npgsql" />