Fix line break algorithm

Now that the test actually does assertions, numerous bugs were
encountered with the algo implementation. Fix algo so all tests pass.

Author: skizzerz

Netwolf.Generator.Test/Netwolf.Generator.Test.csproj

@@ -46,6 +46,10 @@
       <LogicalName>Netwolf.Generator.Test.EastAsianWidth.txt</LogicalName>
       <Visible>false</Visible>
     </EmbeddedResource>
+    <EmbeddedResource Include="..\Netwolf.Unicode\Data\emoji-data.txt">
+      <LogicalName>Netwolf.Generator.Test.emoji-data.txt</LogicalName>
+      <Visible>false</Visible>
+    </EmbeddedResource>
     <EmbeddedResource Include="..\Netwolf.Unicode\Data\HangulSyllableType.txt">
       <LogicalName>Netwolf.Generator.Test.HangulSyllableType.txt</LogicalName>
       <Visible>false</Visible>

Netwolf.Generator.Test/UnicodePropertyTests.cs

@@ -43,6 +43,7 @@ internal static partial class UnicodeProperty
         internal static partial bool IsJoinControl(Rune rune);
         internal static partial bool IsNoncharacterCodePoint(Rune rune);
         internal static partial bool IsDefaultIgnorableCodePoint(Rune rune);
+        internal static partial bool IsExtendedPictographic(Rune rune);
         internal static partial BidiClass GetBidiClass(Rune rune);
         internal static partial HangulSyllableType GetHangulSyllableType(Rune rune);
         internal static partial CombiningClass GetCombiningClass(Rune rune);
@@ -72,6 +73,7 @@ internal static partial class UnicodeProperty
                 new EmbeddedAdditionalText("DerivedCoreProperties.txt"),
                 new EmbeddedAdditionalText("DerivedJoiningType.txt"),
                 new EmbeddedAdditionalText("EastAsianWidth.txt"),
+                new EmbeddedAdditionalText("emoji-data.txt"),
                 new EmbeddedAdditionalText("HangulSyllableType.txt"),
                 new EmbeddedAdditionalText("LineBreak.txt"),
                 new EmbeddedAdditionalText("PropList.txt"),
@@ -82,7 +84,7 @@ internal static partial class UnicodeProperty
 
         var outputDiagnostics = output.GetDiagnostics();
         Assert.IsEmpty(outputDiagnostics, "Diagnostics were created while compiling generated source.");
-        // 9 source files specified above, plus 8 generated files (PropList and DerivedCoreProperties both go to the same file)
+        // 9 source files specified above, plus 8 generated files (PropList/DerivedCoreProperties/emoji-data all go to the same file)
         Assert.AreEqual(17, output.SyntaxTrees.Count());
 
         using MemoryStream assemblyStream = new();

Netwolf.Generator/Unicode/UnicodeDataGenerator.cs

@@ -23,6 +23,7 @@ public class UnicodeDataGenerator : IIncrementalGenerator
         "Join_Control",
         "Noncharacter_Code_Point",
         "Default_Ignorable_Code_Point",
+        "Extended_Pictographic",
     ];
 
     public static StringBuilder GetDecompositionSourceText(ImmutableArray<DecompositionMapping?> mappings)
@@ -269,7 +270,7 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
             .Collect();
 
         var props = context.AdditionalTextsProvider
-            .Where(static text => text.Path.EndsWith("PropList.txt") || text.Path.EndsWith("DerivedCoreProperties.txt"))
+            .Where(static text => text.Path.EndsWith("PropList.txt") || text.Path.EndsWith("DerivedCoreProperties.txt") || text.Path.EndsWith("emoji-data.txt"))
             .Combine(assembly)
             .Where(static model => model.Right == "Netwolf.Unicode")
             .SelectMany(static (model, token) => model.Left.GetText(token)?.ToString().Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries) ?? [])

Netwolf.Test/Unicode/LineBreakTests.cs

@@ -34,12 +34,22 @@ public void Test_split_algorithm(string test, string _)
         }
 
         // this should split on every break opportunity (both optional and mandatory)
-        var lines = LineBreakHelper.SplitText(sb.ToString(), 1, true);
+        var lines = LineBreakHelper.SplitText(sb.ToString(), 1, SplitTextOptions.AllowOverflow | SplitTextOptions.IncludeBreakCharacters);
 
         foreach (var (line, _) in lines)
         {
-            for (var i = 0; i < line.Length - 1; ++i)
+            var enumerator = line.EnumerateRunes();
+            bool first = true;
+
+            while (enumerator.MoveNext())
             {
+                if (first)
+                {
+                    // skip the first character (the number of non-breaks in the string is one fewer than the number of grapheme clusters in the string)
+                    first = false;
+                    continue;
+                }
+
                 actual.Add(false);
             }
 

Netwolf.Transport/Commands/CommandFactory.cs

@@ -292,7 +292,7 @@ public ICommand[] PrepareClientMessage(
             multilineEnabled = false;
         }
 
-        var lines = LineBreakHelper.SplitText(text, maxlen, false);
+        var lines = LineBreakHelper.SplitText(text, maxlen);
 
         string batchId = Guid.NewGuid().ToString();
         int batchLines = 0;