Skip to content

Commit 0595946

Browse files
Nick-NuonNick-Nuon
authored
Merge pull request #33 from simdutf/avx2_new_algo_and_tests_fix
Avx2 new algo and tests fix
2 parents 2e2c0d4 + 0f4d294 commit 0595946

File tree

2 files changed

+48
-40
lines changed

2 files changed

+48
-40
lines changed

‎README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ cd test
2727
dotnet test
2828
```
2929

30+
To see which tests are running, we recommend setting the verbosity level:
31+
32+
```
33+
dotnet test -v d
34+
```
35+
3036
To get a list of available tests, enter the command:
3137

3238
```

‎test/UTF8ValidationTests.cs‎

Lines changed: 42 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,12 @@ public unsafe class Utf8SIMDValidationTests
1313
{
1414

1515

16-
private const int NumTrials = 1000;
16+
private const int NumTrials = 100;
1717
private static readonly RandomUtf8 generator = new RandomUtf8(1234, 1, 1, 1, 1);
1818
private static readonly Random rand = new Random();
1919

2020
// int[] outputLengths = { 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1088, 1152, 1216, 1280, 1344, 1408, 1472, 1536, 1600, 1664, 1728, 1792, 1856, 1920, 1984, 2048, 2112, 2176, 2240, 2304, 2368, 2432, 2496, 2560, 2624, 2688, 2752, 2816, 2880, 2944, 3008, 3072, 3136, 3200, 3264, 3328, 3392, 3456, 3520, 3584, 3648, 3712, 3776, 3840, 3904, 3968, 4032, 4096, 4160, 4224, 4288, 4352, 4416, 4480, 4544, 4608, 4672, 4736, 4800, 4864, 4928, 4992, 5056, 5120, 5184, 5248, 5312, 5376, 5440, 5504, 5568, 5632, 5696, 5760, 5824, 5888, 5952, 6016, 6080, 6144, 6208, 6272, 6336, 6400, 6464, 6528, 6592, 6656, 6720, 6784, 6848, 6912, 6976, 7040, 7104, 7168, 7232, 7296, 7360, 7424, 7488, 7552, 7616, 7680, 7744, 7808, 7872, 7936, 8000, 8064, 8128, 8192, 8256, 8320, 8384, 8448, 8512, 8576, 8640, 8704, 8768, 8832, 8896, 8960, 9024, 9088, 9152, 9216, 9280, 9344, 9408, 9472, 9536, 9600, 9664, 9728, 9792, 9856, 9920, 9984, 10000 };
21-
static int[] outputLengths = { 128, 256,345, 512,968, 1024, 1000 };
21+
static int[] outputLengths = { 128, 345, 1000 };
2222

2323
[Flags]
2424
public enum TestSystemRequirements
@@ -76,7 +76,7 @@ public TestIfCondition(Func<bool> condition, string skipReason)
7676

7777

7878

79-
public void simpleGoodSequences(Utf8ValidationDelegate utf8ValidationDelegate)
79+
private void simpleGoodSequences(Utf8ValidationDelegate utf8ValidationDelegate)
8080
{
8181
string[] goodSequences = {
8282
"a",
@@ -137,15 +137,15 @@ public void simpleGoodSequencesScalar()
137137
// simpleGoodSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
138138
// }
139139

140-
[Fact]
141140
[Trait("Category", "avx")]
141+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
142142
public void simpleGoodSequencesAVX()
143143
{
144144
simpleGoodSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
145145
}
146146

147147

148-
public void BadSequences(Utf8ValidationDelegate utf8ValidationDelegate)
148+
private void BadSequences(Utf8ValidationDelegate utf8ValidationDelegate)
149149
{
150150
string[] badSequences = {
151151
"\xC3\x28",
@@ -225,21 +225,21 @@ public void BadSequencesScalar()
225225
// BadSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
226226
// }
227227

228-
[Fact]
229228
[Trait("Category", "avx")]
229+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
230230
public void BadSequencesAVX()
231231
{
232232
BadSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
233233
}
234234

235235
// this was in the C++ code
236-
public void Node48995Test(Utf8ValidationDelegate utf8ValidationDelegate)
236+
private void Node48995Test(Utf8ValidationDelegate utf8ValidationDelegate)
237237
{
238238
byte[] bad = new byte[] { 0x80 };
239239
Assert.False(ValidateUtf8(bad,utf8ValidationDelegate));
240240
}
241241

242-
public void NoError(Utf8ValidationDelegate utf8ValidationDelegate)
242+
private void NoError(Utf8ValidationDelegate utf8ValidationDelegate)
243243
{
244244
foreach (int outputLength in outputLengths)
245245
{
@@ -295,14 +295,14 @@ public void NoErrorScalar()
295295
// NoError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
296296
// }
297297

298-
[Fact]
299298
[Trait("Category", "avx")]
299+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
300300
public void NoErrorAVX()
301301
{
302302
NoError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
303303
}
304304

305-
public void NoErrorSpecificByteCount(Utf8ValidationDelegate utf8ValidationDelegate)
305+
private void NoErrorSpecificByteCount(Utf8ValidationDelegate utf8ValidationDelegate)
306306
{
307307
RunTestForByteLength(1,utf8ValidationDelegate);
308308
RunTestForByteLength(2,utf8ValidationDelegate);
@@ -366,14 +366,14 @@ public void NoErrorSpecificByteCountScalar()
366366
// NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
367367
// }
368368

369-
[Fact]
370369
[Trait("Category", "avx")]
370+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
371371
public void NoErrorSpecificByteCountAVX()
372372
{
373373
NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
374374
}
375375

376-
public void NoErrorIncompleteThenASCII(Utf8ValidationDelegate utf8ValidationDelegate)
376+
private void NoErrorIncompleteThenASCII(Utf8ValidationDelegate utf8ValidationDelegate)
377377
{
378378
foreach (int outputLength in outputLengths){
379379
for (int trial = 0; trial < NumTrials; trial++)
@@ -442,20 +442,19 @@ public void NoErrorIncompleteThenASCIIScalar()
442442
// NoErrorIncompleteThenASCII(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
443443
// }
444444

445-
[Fact]
446445
[Trait("Category", "avx")]
446+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
447447
public void NoErrorIncompleteThenASCIIAVX()
448448
{
449449
NoErrorIncompleteThenASCII(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
450450
}
451451

452452

453453

454-
public void NoErrorIncompleteAt256Vector(Utf8ValidationDelegate utf8ValidationDelegate)
454+
private void NoErrorIncompleteAt256Vector(Utf8ValidationDelegate utf8ValidationDelegate)
455455
{
456-
// foreach (int outputLength in outputLengths)
456+
foreach (int outputLength in outputLengths)
457457
{
458-
int outputLength = 256;
459458
for (int trial = 0; trial < NumTrials; trial++)
460459
{
461460

@@ -519,14 +518,14 @@ public void NoErrorIncompleteAt256VectorScalar()
519518
// NoErrorIncompleteAt256Vector(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
520519
// }
521520

522-
[Fact]
523521
[Trait("Category", "avx")]
522+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
524523
public void NoErrorIncompleteAt256VectorAVX()
525524
{
526525
NoErrorIncompleteAt256Vector(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
527526
}
528527

529-
public void BadHeaderBits(Utf8ValidationDelegate utf8ValidationDelegate)
528+
private void BadHeaderBits(Utf8ValidationDelegate utf8ValidationDelegate)
530529
{
531530
foreach (int outputLength in outputLengths)
532531
{
@@ -594,14 +593,14 @@ public void BadHeaderBitsScalar()
594593
// NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
595594
// }
596595

597-
[Fact]
598596
[Trait("Category", "avx")]
597+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
599598
public void BadHeaderBitsAVX()
600599
{
601600
BadHeaderBits(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
602601
}
603602

604-
public void TooShortError(Utf8ValidationDelegate utf8ValidationDelegate)
603+
private void TooShortError(Utf8ValidationDelegate utf8ValidationDelegate)
605604
{
606605
foreach (int outputLength in outputLengths)
607606
{
@@ -667,14 +666,14 @@ public void TooShortErrorScalar()
667666
// TooShortError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
668667
// }
669668

670-
[Fact]
671669
[Trait("Category", "avx")]
670+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
672671
public void TooShortErrorAVX()
673672
{
674673
TooShortError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
675674
}
676675

677-
public void TooLongError(Utf8ValidationDelegate utf8ValidationDelegate)
676+
private void TooLongError(Utf8ValidationDelegate utf8ValidationDelegate)
678677
{
679678

680679
foreach (int outputLength in outputLengths)
@@ -740,14 +739,14 @@ public void TooLongErrorScalar()
740739
// TooLongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
741740
// }
742741

743-
[Fact]
744742
[Trait("Category", "avx")]
743+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
745744
public void TooLongErrorAVX()
746745
{
747746
TooLongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
748747
}
749748

750-
public void OverlongError(Utf8ValidationDelegate utf8ValidationDelegate)
749+
private void OverlongError(Utf8ValidationDelegate utf8ValidationDelegate)
751750
{
752751
for (int trial = 0; trial < NumTrials; trial++)
753752
{
@@ -822,15 +821,15 @@ public void OverlongErrorScalar()
822821
// OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
823822
// }
824823

825-
[Fact]
826824
[Trait("Category", "avx")]
825+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
827826
public void OverlongErrorAVX()
828827
{
829828
OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
830829
}
831830

832831

833-
public void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)
832+
private void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)
834833
{
835834
for (int trial = 0; trial < NumTrials; trial++)
836835
{
@@ -909,22 +908,23 @@ public void TooShortErrorAtEndScalar()
909908
// TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
910909
// }
911910

912-
[Fact]
913911
[Trait("Category", "avx")]
912+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
914913
public void TooShortErrorAtEndAVX()
915914
{
916915
TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
917916
}
918917

919-
[Fact]
918+
[Trait("Category", "avx")]
919+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
920920
public void TooShortErrorAtEndAvx2()
921921
{
922922
TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
923923
}
924924

925925

926926
//corresponds to condition 5.4.1 in the paper
927-
public void Invalid0xf50xff(Utf8ValidationDelegate utf8ValidationDelegate)
927+
private void Invalid0xf50xff(Utf8ValidationDelegate utf8ValidationDelegate)
928928
{
929929

930930
var invalidBytes = Enumerable.Range(0xF5, 0x100 - 0xF5).Select(i => (byte)i).ToArray(); // 0xF5 to 0xFF
@@ -976,14 +976,15 @@ public void Invalid0xf50xffScalar()
976976
// Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
977977
// }
978978

979-
[Fact]
980979
[Trait("Category", "avx")]
980+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
981981
public void Invalid0xf50xffAVX()
982982
{
983983
Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
984984
}
985985

986-
[Fact]
986+
[Trait("Category", "avx")]
987+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
987988
public void Invalid0xf50xffAvx2()
988989
{
989990
Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
@@ -1053,7 +1054,7 @@ static void PrintHexAndBinary(byte[] bytes, int highlightIndex = -1)
10531054
}
10541055

10551056

1056-
public void TooLargeError(Utf8ValidationDelegate utf8ValidationDelegate)
1057+
private void TooLargeError(Utf8ValidationDelegate utf8ValidationDelegate)
10571058
{
10581059
foreach (int outputLength in outputLengths)
10591060
{
@@ -1110,15 +1111,15 @@ public void TooLargeErrorScalar()
11101111
// TooLargeError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
11111112
// }
11121113

1113-
[Fact]
11141114
[Trait("Category", "avx")]
1115+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
11151116
public void TooLargeErrorAvx()
11161117
{
11171118
TooLargeError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
11181119
}
11191120

11201121

1121-
public void AsciiPlusContinuationAtEndError(Utf8ValidationDelegate utf8ValidationDelegate)
1122+
private void AsciiPlusContinuationAtEndError(Utf8ValidationDelegate utf8ValidationDelegate)
11221123
{
11231124
foreach (int outputLength in outputLengths)
11241125
{
@@ -1173,20 +1174,21 @@ public void AsciiPlusContinuationAtEndErrorScalar()
11731174
// AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
11741175
// }
11751176

1176-
[Fact]
11771177
[Trait("Category", "avx")]
1178+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
11781179
public void AsciiPlusContinuationAtEndErrorAVX()
11791180
{
11801181
AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
11811182
}
11821183

1183-
[Fact]
1184+
[Trait("Category", "avx")]
1185+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
11841186
public void AsciiPlusContinuationAtEndErrorAvx2()
11851187
{
11861188
AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
11871189
}
11881190

1189-
public void SurrogateErrorTest(Utf8ValidationDelegate utf8ValidationDelegate)
1191+
private void SurrogateErrorTest(Utf8ValidationDelegate utf8ValidationDelegate)
11901192
{
11911193
foreach (int outputLength in outputLengths)
11921194
{
@@ -1253,15 +1255,15 @@ public void SurrogateErrorTestScalar()
12531255
// SurrogateErrorTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
12541256
// }
12551257

1256-
[Fact]
12571258
[Trait("Category", "avx")]
1259+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
12581260
public void SurrogateErrorTestAVX()
12591261
{
12601262
SurrogateErrorTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
12611263
}
12621264

12631265

1264-
public void BruteForceTest(Utf8ValidationDelegate utf8ValidationDelegate)
1266+
private void BruteForceTest(Utf8ValidationDelegate utf8ValidationDelegate)
12651267
{
12661268
foreach (int outputLength in outputLengths)
12671269
{
@@ -1340,8 +1342,8 @@ public void BruteForceTestScalar()
13401342
// BruteForceTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
13411343
// }
13421344

1343-
[Fact]
13441345
[Trait("Category", "avx")]
1346+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
13451347
public void BruteForceTestAVX()
13461348
{
13471349
BruteForceTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);

0 commit comments

Comments
 (0)