Add Unit Test project

.github/workflows/tests.yml

@@ -0,0 +1,18 @@
+name: Build & Test
+
+on:
+    push:
+        branches: [main]
+    pull_request:
+        branches: [main]
+
+jobs:
+    build-test:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-dotnet@v4
+              with:
+                  dotnet-version: 8.0.x
+            - name: Test
+              run: dotnet test -c Release

KokoroSharp.Tests/KokoroSharp.Tests.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+        <OutputType>Exe</OutputType>
+        <IsPackable>false</IsPackable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="TUnit" Version="0.25.6" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\KokoroSharp\KokoroSharp.csproj" />
+    </ItemGroup>
+
+</Project>

KokoroSharp.Tests/TokenizerTests.cs

@@ -0,0 +1,17 @@
+using KokoroSharp.Processing;
+
+namespace KokoroSharp.Tests;
+
+public class TokenizerTests {
+    [Test]
+    [Arguments("$1", "1 dollar")]
+    [Arguments("$1.50", "1 dollar 50")]
+    [Arguments("$ 1.50", "1 dollar 50")]
+    [Arguments("1€", "1 euro")]
+    [Arguments("1,75 €", "1 euro 75")]
+    [Arguments("1,75€", "1 euro 75")]
+    [Arguments("3.1415", "3 point 1 4 1 5")]
+    public async Task PreprocessText(string input, string expected) {
+        await Assert.That(Tokenizer.PreprocessText(input)).IsEqualTo(expected);
+    }
+}

KokoroSharp.sln

@@ -5,6 +5,8 @@ VisualStudioVersion = 17.12.35506.116
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KokoroSharp", "KokoroSharp\KokoroSharp.csproj", "{81F1B2B4-923A-4AD5-BAC9-72EA1620E162}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KokoroSharp.Tests", "KokoroSharp.Tests\KokoroSharp.Tests.csproj", "{B0AB7C0B-D1C1-447C-9EB5-5CC9CB9E8943}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -15,6 +17,10 @@ Global
 		{81F1B2B4-923A-4AD5-BAC9-72EA1620E162}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{81F1B2B4-923A-4AD5-BAC9-72EA1620E162}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{81F1B2B4-923A-4AD5-BAC9-72EA1620E162}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B0AB7C0B-D1C1-447C-9EB5-5CC9CB9E8943}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B0AB7C0B-D1C1-447C-9EB5-5CC9CB9E8943}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B0AB7C0B-D1C1-447C-9EB5-5CC9CB9E8943}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B0AB7C0B-D1C1-447C-9EB5-5CC9CB9E8943}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

KokoroSharp/KokoroSharp.csproj

@@ -32,4 +32,9 @@
     <ItemGroup>
         <None Include="KokoroSharp.targets" Pack="true" PackagePath="buildTransitive/" />
     </ItemGroup>
+
+    <ItemGroup>
+        <InternalsVisibleTo Include="KokoroSharp.Tests" />
+    </ItemGroup>
+
 </Project>

KokoroSharp/Processing/Tokenizer.cs

@@ -81,7 +81,7 @@ public static string Phonemize_Internal(string text, out string originalSegments
 
     /// <summary> Normalizes the input text to what the Kokoro model would expect to see, preparing it for phonemization. </summary>
     /// <remarks> In addition, converts various "written" text to "spoken" form (e.g. $1 --> "one dollar" instead of "dollar one". </remarks>
-    internal static string PreprocessText(string text, string langCode) {
+    internal static string PreprocessText(string text, string langCode = "en-us") {
         text = HeaderLink().Replace(text, "$1"); // Discard links appearing in `[Header](link)` format.
         text = HeaderImgLink().Replace(text, "$1$2"); // And in [Header[(img](link)]
         text = Money().Replace(text, "$2 $1 $3"); // Convert money amounts like "$1.50" to "1 $ 50".