Skip to content

Commit 48e9db6

Browse files
lemirelemire
committed
renaming and tweaks
1 parent c0f1a09 commit 48e9db6

File tree

3 files changed

+116
-35
lines changed

3 files changed

+116
-35
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ To run specific tests, it is helpful to use the filter parameter:
4444

4545

4646
```
47-
dotnet test --filter TooShortErrorAVX
47+
dotnet test --filter TooShortErrorAvx2
4848
```
4949

5050
Or to target specific categories:

‎src/UTF8.cs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -687,7 +687,7 @@ public unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust(
687687
if (processedLength < inputLength)
688688
{
689689

690-
byte* invalidBytePointer = SimdUnicode.UTF8.RewindAndValidateWithErrors(32, pInputBuffer + processedLength, inputLength - processedLength, ref TailUtf16CodeUnitCountAdjustment, ref TailScalarCodeUnitCountAdjustment);
690+
byte* invalidBytePointer = SimdUnicode.UTF8.RewindAndValidateWithErrors(processedLength, pInputBuffer + processedLength, inputLength - processedLength, ref TailUtf16CodeUnitCountAdjustment, ref TailScalarCodeUnitCountAdjustment);
691691
if (invalidBytePointer != pInputBuffer + inputLength)
692692
{
693693
utf16CodeUnitCountAdjustment = TempUtf16CodeUnitCountAdjustment + TailUtf16CodeUnitCountAdjustment;
@@ -834,7 +834,7 @@ public unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust(
834834
// hardware:
835835
if (AdvSimd.Arm64.MaxAcross(Vector128.AsUInt32(error)).ToScalar() != 0)
836836
{
837-
int off = processedLength > 32 ? processedLength - 32 : processedLength;// this does not backup ff processedlength = 32
837+
int off = processedLength > 32 ? processedLength - 32 : processedLength;// this does not backup off processedlength = 32
838838
byte* invalidBytePointer = SimdUnicode.UTF8.RewindAndValidateWithErrors(off, pInputBuffer + processedLength, inputLength - processedLength, ref TailUtf16CodeUnitCountAdjustment, ref TailScalarCodeUnitCountAdjustment);
839839
utf16CodeUnitCountAdjustment = TailUtf16CodeUnitCountAdjustment;
840840
scalarCountAdjustment = TailScalarCodeUnitCountAdjustment;
@@ -866,13 +866,12 @@ public unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust(
866866
}
867867
// We have processed all the blocks using SIMD, we need to process the remaining bytes.
868868
// Process the remaining bytes with the scalar function
869-
870869
// worst possible case is 4 bytes, where we need to backtrack 3 bytes
871870
// 11110xxxx 10xxxxxx 10xxxxxx 10xxxxxx <== we might be pointing at the last byte
872871
if (processedLength < inputLength)
873872
{
874873

875-
byte* invalidBytePointer = SimdUnicode.UTF8.RewindAndValidateWithErrors(32, pInputBuffer + processedLength, inputLength - processedLength, ref TailUtf16CodeUnitCountAdjustment, ref TailScalarCodeUnitCountAdjustment);
874+
byte* invalidBytePointer = SimdUnicode.UTF8.RewindAndValidateWithErrors(processedLength, pInputBuffer + processedLength, inputLength - processedLength, ref TailUtf16CodeUnitCountAdjustment, ref TailScalarCodeUnitCountAdjustment);
876875
if (invalidBytePointer != pInputBuffer + inputLength)
877876
{
878877
utf16CodeUnitCountAdjustment = TempUtf16CodeUnitCountAdjustment + TailUtf16CodeUnitCountAdjustment;

‎test/UTF8ValidationTests.cs‎

Lines changed: 112 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -114,11 +114,17 @@ public void simpleGoodSequencesScalar()
114114

115115
[Trait("Category", "avx")]
116116
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
117-
public void simpleGoodSequencesAVX()
117+
public void simpleGoodSequencesAvx2()
118118
{
119119
simpleGoodSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
120120
}
121121

122+
[Trait("Category", "arm64")]
123+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
124+
public void simpleGoodSequencesArm64()
125+
{
126+
simpleGoodSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
127+
}
122128

123129
private void BadSequences(Utf8ValidationDelegate utf8ValidationDelegate)
124130
{
@@ -177,11 +183,18 @@ public void BadSequencesScalar()
177183

178184
[Trait("Category", "avx")]
179185
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
180-
public void BadSequencesAVX()
186+
public void BadSequencesAvx2()
181187
{
182188
BadSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
183189
}
184190

191+
[Trait("Category", "arm64")]
192+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
193+
public void BadSequencesArm64()
194+
{
195+
BadSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
196+
}
197+
185198
// this was in the C++ code
186199
private void Node48995Test(Utf8ValidationDelegate utf8ValidationDelegate)
187200
{
@@ -222,11 +235,18 @@ public void NoErrorScalar()
222235

223236
[Trait("Category", "avx")]
224237
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
225-
public void NoErrorAVX()
238+
public void NoErrorAvx2()
226239
{
227240
NoError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
228241
}
229242

243+
[Trait("Category", "arm64")]
244+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
245+
public void NoErrorArm64()
246+
{
247+
NoError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
248+
}
249+
230250
private void NoErrorSpecificByteCount(Utf8ValidationDelegate utf8ValidationDelegate)
231251
{
232252
RunTestForByteLength(1,utf8ValidationDelegate);
@@ -268,11 +288,17 @@ public void NoErrorSpecificByteCountScalar()
268288

269289
[Trait("Category", "avx")]
270290
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
271-
public void NoErrorSpecificByteCountAVX()
291+
public void NoErrorSpecificByteCountAvx2()
272292
{
273293
NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
274294
}
275295

296+
[Trait("Category", "arm64")]
297+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
298+
public void NoErrorSpecificByteCountArm64()
299+
{
300+
NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
301+
}
276302
private void NoErrorIncompleteThenASCII(Utf8ValidationDelegate utf8ValidationDelegate)
277303
{
278304
foreach (int outputLength in outputLengths){
@@ -319,12 +345,18 @@ public void NoErrorIncompleteThenASCIIScalar()
319345

320346
[Trait("Category", "avx")]
321347
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
322-
public void NoErrorIncompleteThenASCIIAVX()
348+
public void NoErrorIncompleteThenASCIIAvx2()
323349
{
324350
NoErrorIncompleteThenASCII(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
325351
}
326352

327353

354+
[Trait("Category", "arm64")]
355+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
356+
public void NoErrorIncompleteThenASCIIArm64()
357+
{
358+
NoErrorIncompleteThenASCII(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
359+
}
328360

329361
private void NoErrorIncompleteAt256Vector(Utf8ValidationDelegate utf8ValidationDelegate)
330362
{
@@ -370,11 +402,18 @@ public void NoErrorIncompleteAt256VectorScalar()
370402

371403
[Trait("Category", "avx")]
372404
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
373-
public void NoErrorIncompleteAt256VectorAVX()
405+
public void NoErrorIncompleteAt256VectorAvx2()
374406
{
375407
NoErrorIncompleteAt256Vector(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
376408
}
377409

410+
[Trait("Category", "arm64")]
411+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
412+
public void NoErrorIncompleteAt256VectorArm64()
413+
{
414+
NoErrorIncompleteAt256Vector(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
415+
}
416+
378417
private void BadHeaderBits(Utf8ValidationDelegate utf8ValidationDelegate)
379418
{
380419
foreach (int outputLength in outputLengths)
@@ -419,11 +458,18 @@ public void BadHeaderBitsScalar()
419458

420459
[Trait("Category", "avx")]
421460
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
422-
public void BadHeaderBitsAVX()
461+
public void BadHeaderBitsAvx2()
423462
{
424463
BadHeaderBits(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
425464
}
426465

466+
[Trait("Category", "arm64")]
467+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
468+
public void BadHeaderBitsArm64()
469+
{
470+
BadHeaderBits(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
471+
}
472+
427473
private void TooShortError(Utf8ValidationDelegate utf8ValidationDelegate)
428474
{
429475
foreach (int outputLength in outputLengths)
@@ -467,11 +513,18 @@ public void TooShortErrorScalar()
467513

468514
[Trait("Category", "avx")]
469515
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
470-
public void TooShortErrorAVX()
516+
public void TooShortErrorAvx2()
471517
{
472518
TooShortError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
473519
}
474520

521+
[Trait("Category", "arm64")]
522+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
523+
public void TooShortErrorArm64()
524+
{
525+
TooShortError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
526+
}
527+
475528
private void TooLongError(Utf8ValidationDelegate utf8ValidationDelegate)
476529
{
477530

@@ -515,11 +568,18 @@ public void TooLongErrorScalar()
515568

516569
[Trait("Category", "avx")]
517570
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
518-
public void TooLongErrorAVX()
571+
public void TooLongErrorAvx2()
519572
{
520573
TooLongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
521574
}
522575

576+
[Trait("Category", "arm64")]
577+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
578+
public void TooLongErrorArm64()
579+
{
580+
TooLongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
581+
}
582+
523583
private void OverlongError(Utf8ValidationDelegate utf8ValidationDelegate)
524584
{
525585
for (int trial = 0; trial < NumTrials; trial++)
@@ -570,11 +630,11 @@ public void OverlongErrorScalar()
570630
OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar);
571631
}
572632

573-
[Trait("Category", "avx")]
574-
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
575-
public void OverlongErrorAVX()
633+
[Trait("Category", "arm64")]
634+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
635+
public void OverlongErrorArm64()
576636
{
577-
OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
637+
OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
578638
}
579639

580640

@@ -632,12 +692,6 @@ public void TooShortErrorAtEndScalar()
632692
TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar);
633693
}
634694

635-
[Trait("Category", "avx")]
636-
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
637-
public void TooShortErrorAtEndAVX()
638-
{
639-
TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
640-
}
641695

642696
[Trait("Category", "avx")]
643697
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
@@ -646,6 +700,12 @@ public void TooShortErrorAtEndAvx2()
646700
TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
647701
}
648702

703+
[Trait("Category", "arm64")]
704+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
705+
public void TooShortErrorAtEndArm64()
706+
{
707+
TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
708+
}
649709

650710
//corresponds to condition 5.4.1 in the paper
651711
private void Invalid0xf50xff(Utf8ValidationDelegate utf8ValidationDelegate)
@@ -675,18 +735,20 @@ public void Invalid0xf50xffScalar()
675735
Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar);
676736
}
677737

738+
678739
[Trait("Category", "avx")]
679740
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
680-
public void Invalid0xf50xffAVX()
741+
public void Invalid0xf50xffAvx2()
681742
{
682743
Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
683744
}
684745

685-
[Trait("Category", "avx")]
686-
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
687-
public void Invalid0xf50xffAvx2()
746+
747+
[Trait("Category", "arm64")]
748+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
749+
public void Invalid0xf50xffArm64()
688750
{
689-
Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
751+
Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
690752
}
691753

692754
// helper function for debugging: it prints a green byte every 32 bytes and a red byte at a given index
@@ -793,6 +855,13 @@ public void TooLargeErrorAvx()
793855
}
794856

795857

858+
[Trait("Category", "arm64")]
859+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
860+
public void TooLargeErrorArm64()
861+
{
862+
TooLargeError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
863+
}
864+
796865
private void AsciiPlusContinuationAtEndError(Utf8ValidationDelegate utf8ValidationDelegate)
797866
{
798867
foreach (int outputLength in outputLengths)
@@ -823,11 +892,11 @@ public void AsciiPlusContinuationAtEndErrorScalar()
823892
AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar);
824893
}
825894

826-
[Trait("Category", "avx")]
827-
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
828-
public void AsciiPlusContinuationAtEndErrorAVX()
895+
[Trait("Category", "arm64")]
896+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
897+
public void AsciiPlusContinuationAtEndErrorArm64()
829898
{
830-
AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
899+
AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
831900
}
832901

833902
[Trait("Category", "avx")]
@@ -881,11 +950,17 @@ public void SurrogateErrorTestScalar()
881950

882951
[Trait("Category", "avx")]
883952
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
884-
public void SurrogateErrorTestAVX()
953+
public void SurrogateErrorTestAvx2()
885954
{
886955
SurrogateErrorTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
887956
}
888957

958+
[Trait("Category", "arm64")]
959+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
960+
public void SurrogateErrorTestArm64()
961+
{
962+
SurrogateErrorTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
963+
}
889964

890965
private void BruteForceTest(Utf8ValidationDelegate utf8ValidationDelegate)
891966
{
@@ -943,12 +1018,19 @@ public void BruteForceTestScalar()
9431018

9441019
[Trait("Category", "avx")]
9451020
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
946-
public void BruteForceTestAVX()
1021+
public void BruteForceTestAvx2()
9471022
{
9481023
BruteForceTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
9491024
}
9501025

9511026

1027+
[Trait("Category", "arm64")]
1028+
[FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)]
1029+
public void BruteForceTestArm64()
1030+
{
1031+
BruteForceTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
1032+
}
1033+
9521034
// credit: based on code from Google Fuchsia (Apache Licensed)
9531035
public static bool ValidateUtf8Fuschia(byte[] data)
9541036
{

0 commit comments

Comments
 (0)