Skip to content

Commit 8380baf

Browse files
jthackraymomchil-velikovLukacma
authored
[AArch64] Add intrinsics for SME FP8 FVDOT, FVDOTB and FVDOTT intrinsics (#119922)
Add support for the following SME 8 bit floating-point dot-product intrinsics: ``` // Only if __ARM_FEATURE_SME_F8F16 != 0 void svvdot_lane_za16[_mf8]_vg1x2_fpm(uint32_t slice, svmfloat8x2_t zn, svmfloat8_t zm, uint64_t imm_idx, fpm_t fpm) __arm_streaming __arm_inout("za"); // Only if __ARM_FEATURE_SME_F8F32 != 0 void svvdott_lane_za32[_mf8]_vg1x4_fpm(uint32_t slice, svmfloat8x2_t zn, svmfloat8_t zm, uint64_t imm_idx, fpm_t fpm) __arm_streaming __arm_inout("za"); void svvdotb_lane_za32[_mf8]_vg1x4_fpm(uint32_t slice, svmfloat8x2_t zn, svmfloat8_t zm, uint64_t imm_idx, fpm_t fpm) __arm_streaming __arm_inout("za"); ``` --------- Co-authored-by: Momchil Velikov <momchil.velikov@arm.com> Co-authored-by: Marian Lukac <marian.lukac@arm.com>
1 parent 3c3094b commit 8380baf

File tree

7 files changed

+187
-6
lines changed

7 files changed

+187
-6
lines changed

‎clang/include/clang/Basic/arm_sme.td‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -749,6 +749,9 @@ let SMETargetGuard = "sme-f8f32" in {
749749
def SVDOT_LANE_FP8_ZA32_VG1x2 : Inst<"svdot_lane_za32[_mf8]_vg1x2_fpm", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fdot_lane_za32_vg1x2", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_3>]>;
750750
def SVDOT_LANE_FP8_ZA32_VG1x4 : Inst<"svdot_lane_za32[_mf8]_vg1x4_fpm", "vm4di>", "m", MergeNone, "aarch64_sme_fp8_fdot_lane_za32_vg1x4", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_3>]>;
751751

752+
def SVVDOTB_LANE_FP8_ZA32_VG1x4 : Inst<"svvdotb_lane_za32[_mf8]_vg1x4_fpm", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fvdotb_lane_za32_vg1x4", [IsOverloadNone, IsStreaming, IsInOutZA, SetsFPMR], [ImmCheck<3, ImmCheck0_3>]>;
753+
def SVVDOTT_LANE_FP8_ZA32_VG1x4 : Inst<"svvdott_lane_za32[_mf8]_vg1x4_fpm", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fvdott_lane_za32_vg1x4", [IsOverloadNone, IsStreaming, IsInOutZA, SetsFPMR], [ImmCheck<3, ImmCheck0_3>]>;
754+
752755
def SVDOT_SINGLE_FP8_ZA32_VG1x2 : Inst<"svdot[_single]_za32[_mf8]_vg1x2_fpm", "vm2d>", "m", MergeNone, "aarch64_sme_fp8_fdot_single_za32_vg1x2", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
753756
def SVDOT_SINGLE_FP8_ZA32_VG1x4 : Inst<"svdot[_single]_za32[_mf8]_vg1x4_fpm", "vm4d>", "m", MergeNone, "aarch64_sme_fp8_fdot_single_za32_vg1x4", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
754757

@@ -760,6 +763,8 @@ let SMETargetGuard = "sme-f8f16" in {
760763
def SVDOT_LANE_FP8_ZA16_VG1x2 : Inst<"svdot_lane_za16[_mf8]_vg1x2_fpm", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fdot_lane_za16_vg1x2", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>;
761764
def SVDOT_LANE_FP8_ZA16_VG1x4 : Inst<"svdot_lane_za16[_mf8]_vg1x4_fpm", "vm4di>", "m", MergeNone, "aarch64_sme_fp8_fdot_lane_za16_vg1x4", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>;
762765

766+
def SVVDOT_LANE_FP8_ZA16_VG1x2 : Inst<"svvdot_lane_za16[_mf8]_vg1x2_fpm", "vm2di>", "m", MergeNone, "aarch64_sme_fp8_fvdot_lane_za16_vg1x2", [IsOverloadNone, IsStreaming, IsInOutZA, SetsFPMR], [ImmCheck<3, ImmCheck0_7>]>;
767+
763768
def SVDOT_SINGLE_FP8_ZA16_VG1x2 : Inst<"svdot[_single]_za16[_mf8]_vg1x2_fpm", "vm2d>", "m", MergeNone, "aarch64_sme_fp8_fdot_single_za16_vg1x2", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
764769
def SVDOT_SINGLE_FP8_ZA16_VG1x4 : Inst<"svdot[_single]_za16[_mf8]_vg1x4_fpm", "vm4d>", "m", MergeNone, "aarch64_sme_fp8_fdot_single_za16_vg1x4", [IsStreaming, IsInOutZA, SetsFPMR, IsOverloadNone], []>;
765770

‎clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_fp8_fvdot.c‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4
2+
3+
// RUN: %clang_cc1 -triple aarch64 -target-feature +sme -target-feature +sme2 -target-feature +sme-f8f16 -target-feature +sme-f8f32 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes mem2reg,instcombine,tailcallelim | FileCheck %s
4+
// RUN: %clang_cc1 -triple aarch64 -target-feature +sme -target-feature +sme2 -target-feature +sme-f8f16 -target-feature +sme-f8f32 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
5+
// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme -target-feature +sme2 -target-feature +sme-f8f16 -target-feature +sme-f8f32 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes mem2reg,instcombine,tailcallelim | FileCheck %s
6+
// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sme -target-feature +sme2 -target-feature +sme-f8f16 -target-feature +sme-f8f32 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
7+
// RUN: %clang_cc1 -triple aarch64 -target-feature +sme -target-feature +sme2 -target-feature +sme-f8f16 -target-feature +sme-f8f32 -target-feature -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
8+
9+
// REQUIRES: aarch64-registered-target
10+
11+
#include <arm_sme.h>
12+
13+
#ifdef SVE_OVERLOADED_FORMS
14+
#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3) A1##A3
15+
#else
16+
#define SVE_ACLE_FUNC(A1, A2, A3) A1##A2##A3
17+
#endif
18+
19+
// CHECK-LABEL: define dso_local void @test_svvdot_lane_za16_mf8_vg1x2(
20+
// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0:[0-9]+]] {
21+
// CHECK-NEXT: entry:
22+
// CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
23+
// CHECK-NEXT: tail call void @llvm.aarch64.sme.fp8.fvdot.lane.za16.vg1x2(i32 [[SLICE]], <vscale x 16 x i8> [[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8> [[ZM]], i32 7)
24+
// CHECK-NEXT: ret void
25+
//
26+
// CPP-CHECK-LABEL: define dso_local void @_Z31test_svvdot_lane_za16_mf8_vg1x2j13svmfloat8x2_tu13__SVMfloat8_tm(
27+
// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0:[0-9]+]] {
28+
// CPP-CHECK-NEXT: entry:
29+
// CPP-CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
30+
// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fp8.fvdot.lane.za16.vg1x2(i32 [[SLICE]], <vscale x 16 x i8> [[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8> [[ZM]], i32 7)
31+
// CPP-CHECK-NEXT: ret void
32+
//
33+
void test_svvdot_lane_za16_mf8_vg1x2(uint32_t slice, svmfloat8x2_t zn,
34+
svmfloat8_t zm,
35+
fpm_t fpmr) __arm_streaming
36+
__arm_inout("za") {
37+
SVE_ACLE_FUNC(svvdot_lane_za16, _mf8, _vg1x2_fpm)(slice, zn, zm, 7, fpmr);
38+
}
39+
40+
// CHECK-LABEL: define dso_local void @test_svvdotb_lane_za32_mf8_vg1x4(
41+
// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
42+
// CHECK-NEXT: entry:
43+
// CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
44+
// CHECK-NEXT: tail call void @llvm.aarch64.sme.fp8.fvdotb.lane.za32.vg1x4(i32 [[SLICE]], <vscale x 16 x i8> [[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8> [[ZM]], i32 3)
45+
// CHECK-NEXT: ret void
46+
//
47+
// CPP-CHECK-LABEL: define dso_local void @_Z32test_svvdotb_lane_za32_mf8_vg1x4j13svmfloat8x2_tu13__SVMfloat8_tm(
48+
// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
49+
// CPP-CHECK-NEXT: entry:
50+
// CPP-CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
51+
// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fp8.fvdotb.lane.za32.vg1x4(i32 [[SLICE]], <vscale x 16 x i8> [[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8> [[ZM]], i32 3)
52+
// CPP-CHECK-NEXT: ret void
53+
//
54+
void test_svvdotb_lane_za32_mf8_vg1x4(uint32_t slice, svmfloat8x2_t zn,
55+
svmfloat8_t zm,
56+
fpm_t fpmr) __arm_streaming
57+
__arm_inout("za") {
58+
SVE_ACLE_FUNC(svvdotb_lane_za32, _mf8, _vg1x4_fpm)(slice, zn, zm, 3, fpmr);
59+
}
60+
61+
// CHECK-LABEL: define dso_local void @test_svvdott_lane_za32_mf8_vg1x4(
62+
// CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
63+
// CHECK-NEXT: entry:
64+
// CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
65+
// CHECK-NEXT: tail call void @llvm.aarch64.sme.fp8.fvdott.lane.za32.vg1x4(i32 [[SLICE]], <vscale x 16 x i8> [[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8> [[ZM]], i32 3)
66+
// CHECK-NEXT: ret void
67+
//
68+
// CPP-CHECK-LABEL: define dso_local void @_Z32test_svvdott_lane_za32_mf8_vg1x4j13svmfloat8x2_tu13__SVMfloat8_tm(
69+
// CPP-CHECK-SAME: i32 noundef [[SLICE:%.*]], <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], i64 noundef [[FPMR:%.*]]) #[[ATTR0]] {
70+
// CPP-CHECK-NEXT: entry:
71+
// CPP-CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
72+
// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.fp8.fvdott.lane.za32.vg1x4(i32 [[SLICE]], <vscale x 16 x i8> [[ZN_COERCE0]], <vscale x 16 x i8> [[ZN_COERCE1]], <vscale x 16 x i8> [[ZM]], i32 3)
73+
// CPP-CHECK-NEXT: ret void
74+
//
75+
void test_svvdott_lane_za32_mf8_vg1x4(uint32_t slice, svmfloat8x2_t zn,
76+
svmfloat8_t zm,
77+
fpm_t fpmr) __arm_streaming
78+
__arm_inout("za") {
79+
SVE_ACLE_FUNC(svvdott_lane_za32, _mf8, _vg1x4_fpm)(slice, zn, zm, 3, fpmr);
80+
}

‎clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_fp8_fvdot.c‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -verify -emit-llvm -o - %s
2+
3+
// REQUIRES: aarch64-registered-target
4+
5+
#include <arm_sme.h>
6+
7+
void test_features(uint32_t slice, fpm_t fpmr, svmfloat8x2_t zn,
8+
svmfloat8_t zm) __arm_streaming __arm_inout("za") {
9+
// expected-error@+1 {{'svvdot_lane_za16_mf8_vg1x2_fpm' needs target feature sme,sme-f8f16}}
10+
svvdot_lane_za16_mf8_vg1x2_fpm(slice, zn, zm, 7, fpmr);
11+
// expected-error@+1 {{'svvdotb_lane_za32_mf8_vg1x4_fpm' needs target feature sme,sme-f8f32}}
12+
svvdotb_lane_za32_mf8_vg1x4_fpm(slice, zn, zm, 3, fpmr);
13+
// expected-error@+1 {{'svvdott_lane_za32_mf8_vg1x4_fpm' needs target feature sme,sme-f8f32}}
14+
svvdott_lane_za32_mf8_vg1x4_fpm(slice, zn, zm, 3, fpmr);
15+
}
16+
17+
void test_imm(uint32_t slice, fpm_t fpmr, svmfloat8x2_t zn,
18+
svmfloat8_t zm) __arm_streaming __arm_inout("za") {
19+
// expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
20+
svvdot_lane_za16_mf8_vg1x2_fpm(slice, zn, zm, -1, fpmr);
21+
// expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
22+
svvdotb_lane_za32_mf8_vg1x4_fpm(slice, zn, zm, -1, fpmr);
23+
// expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
24+
svvdott_lane_za32_mf8_vg1x4_fpm(slice, zn, zm, -1, fpmr);
25+
26+
// expected-error@+1{{argument value 8 is outside the valid range [0, 7]}}
27+
svvdot_lane_za16_mf8_vg1x2_fpm(slice, zn, zm, 8, fpmr);
28+
// expected-error@+1{{argument value 4 is outside the valid range [0, 3]}}
29+
svvdotb_lane_za32_mf8_vg1x4_fpm(slice, zn, zm, 4, fpmr);
30+
// expected-error@+1{{argument value 4 is outside the valid range [0, 3]}}
31+
svvdott_lane_za32_mf8_vg1x4_fpm(slice, zn, zm, 4, fpmr);
32+
}

‎llvm/include/llvm/IR/IntrinsicsAArch64.td‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3904,6 +3904,11 @@ class SME2_FP8_FDOT_MULTI_VG1x4 :
39043904
def int_aarch64_sme_fp8_fdot_lane_za32_vg1x2 : SME2_FP8_FDOT_LANE_VG1x2;
39053905
def int_aarch64_sme_fp8_fdot_lane_za32_vg1x4 : SME2_FP8_FDOT_LANE_VG1x4;
39063906

3907+
def int_aarch64_sme_fp8_fvdot_lane_za16_vg1x2 : SME2_FP8_FDOT_LANE_VG1x2;
3908+
3909+
def int_aarch64_sme_fp8_fvdotb_lane_za32_vg1x4 : SME2_FP8_FDOT_LANE_VG1x2;
3910+
def int_aarch64_sme_fp8_fvdott_lane_za32_vg1x4 : SME2_FP8_FDOT_LANE_VG1x2;
3911+
39073912
def int_aarch64_sme_fp8_fdot_single_za16_vg1x2 : SME2_FP8_FDOT_SINGLE_VG1x2;
39083913
def int_aarch64_sme_fp8_fdot_single_za16_vg1x4 : SME2_FP8_FDOT_SINGLE_VG1x4;
39093914

‎llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -985,8 +985,8 @@ def LUTI4_S_4ZZT2Z : sme2_luti4_vector_vg4_strided<0b00, 0b00, "luti4">;
985985
} //[HasSME2p1, HasSME_LUTv2]
986986

987987
let Predicates = [HasSMEF8F16] in {
988-
defm FVDOT_VG2_M2ZZI_BtoH : sme2p1_multi_vec_array_vg2_index_f8f16<"fvdot", 0b11, 0b110, ZZ_b_mul_r, ZPR4b8>;
989-
defm FDOT_VG2_M2ZZI_BtoH : sme2_fp8_fdot_index_za16_vg1x2<"fdot", int_aarch64_sme_fp8_fdot_lane_za16_vg1x2>;
988+
defm FVDOT_VG2_M2ZZI_BtoH : sme2_fp8_fdot_index_za16_vg1x2<"fvdot", 0b110, int_aarch64_sme_fp8_fvdot_lane_za16_vg1x2>;
989+
defm FDOT_VG2_M2ZZI_BtoH : sme2_fp8_fdot_index_za16_vg1x2<"fdot", 0b010, int_aarch64_sme_fp8_fdot_lane_za16_vg1x2>;
990990
defm FDOT_VG4_M4ZZI_BtoH : sme2_fp8_fdot_index_za16_vg1x4<"fdot", int_aarch64_sme_fp8_fdot_lane_za16_vg1x4>;
991991

992992
defm FDOT_VG2_M2ZZ_BtoH : sme2_fp8_fdot_single_vg1x2<"fdot", 0b0010001, MatrixOp16, int_aarch64_sme_fp8_fdot_single_za16_vg1x2>;
@@ -1016,8 +1016,8 @@ defm FDOT_VG4_M4ZZ_BtoS : sme2_fp8_fdot_single_vg1x4<"fdot", 0b0110011, MatrixO
10161016
defm FDOT_VG2_M2Z2Z_BtoS : sme2_fp8_fdot_multi_vg1x2 <"fdot", 0b0100110, MatrixOp32, int_aarch64_sme_fp8_fdot_multi_za32_vg1x2>;
10171017
defm FDOT_VG4_M4Z4Z_BtoS : sme2_fp8_fdot_multi_vg1x4 <"fdot", 0b0100110, MatrixOp32, int_aarch64_sme_fp8_fdot_multi_za32_vg1x4>;
10181018

1019-
def FVDOTB_VG4_M2ZZI_BtoS : sme2_fp8_multi_vec_array_vg4_index<"fvdotb", 0b0>;
1020-
def FVDOTT_VG4_M2ZZI_BtoS : sme2_fp8_multi_vec_array_vg4_index<"fvdott", 0b1>;
1019+
defm FVDOTB_VG4_M2ZZI_BtoS : sme2_fp8_fdotv_index_za32_vg1x4<"fvdotb", 0b0, int_aarch64_sme_fp8_fvdotb_lane_za32_vg1x4>;
1020+
defm FVDOTT_VG4_M2ZZI_BtoS : sme2_fp8_fdotv_index_za32_vg1x4<"fvdott", 0b1, int_aarch64_sme_fp8_fvdott_lane_za32_vg1x4>;
10211021

10221022
defm FMLALL_MZZI_BtoS : sme2_mla_ll_array_index_32b<"fmlall", 0b01, 0b000, null_frag>;
10231023
defm FMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"fmlall", 0b10, 0b100, null_frag>;

‎llvm/lib/Target/AArch64/SMEInstrFormats.td‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5797,9 +5797,9 @@ multiclass sme2_fmop4a_fp8_fp16_2way<string mnemonic> {
57975797

57985798
// FP8 SME FDOT instructions
57995799

5800-
multiclass sme2_fp8_fdot_index_za16_vg1x2<string mnemonic,
5800+
multiclass sme2_fp8_fdot_index_za16_vg1x2<string mnemonic, bits<3> op,
58015801
SDPatternOperator intrinsic> {
5802-
def NAME : sme2_multi_vec_array_vg2_index<0b11, {0b0,?,?,0b10,?}, MatrixOp16,
5802+
def NAME : sme2_multi_vec_array_vg2_index<0b11, {op{2},?,?,op{1-0},?}, MatrixOp16,
58035803
ZZ_b_mul_r, ZPR4b8,
58045804
VectorIndexH32b_timm, mnemonic>,
58055805
SMEPseudo2Instr<NAME, 1>{
@@ -5883,6 +5883,15 @@ multiclass sme2_fp8_fdot_index_za32_vg1x4<string mnemonic,
58835883
def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, ZPR4b8, nxv16i8, VectorIndexS32b_timm, tileslice16>;
58845884
}
58855885

5886+
multiclass sme2_fp8_fdotv_index_za32_vg1x4<string mnemonic, bit T, SDPatternOperator intrinsic> {
5887+
def NAME : sme2_fp8_multi_vec_array_vg4_index<mnemonic, T>,
5888+
SMEPseudo2Instr<NAME, 1>;
5889+
5890+
def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, ZZ_b_mul_r, ZPR4b8, VectorIndexS32b_timm, SMEMatrixArray>;
5891+
5892+
def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, ZPR4b8, nxv16i8, VectorIndexS32b_timm, tileslice16>;
5893+
}
5894+
58865895
multiclass sme2_fp8_fdot_single_vg1x2<string mnemonic, bits<7> op,
58875896
MatrixOperand matrix_op,
58885897
SDPatternOperator intrinsic> {

‎llvm/test/CodeGen/AArch64/sme2-intrinsics-fp8-fvdot.ll‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "// kill:" --version 4
2+
; RUN: llc -force-streaming < %s | FileCheck %s
3+
target triple = "aarch64-linux"
4+
5+
define void @test_fvdot16_1x2_indexed(i32 %slice.0,
6+
; CHECK-LABEL: test_fvdot16_1x2_indexed:
7+
; CHECK: // %bb.0:
8+
; CHECK: mov w8, w0
9+
; CHECK: fvdot za.h[w8, 7, vgx2], { z0.b, z1.b }, z2.b[3]
10+
; CHECK: ret
11+
<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2,
12+
<vscale x 16 x i8> %zm) #0 {
13+
%slice = add i32 %slice.0, 7
14+
call void @llvm.aarch64.sme.fp8.fvdot.lane.za16.vg1x2(i32 %slice,
15+
<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2,
16+
<vscale x 16 x i8> %zm, i32 3)
17+
ret void
18+
}
19+
20+
define void @test_fvdot32_bottom_1x4_indexed(i32 %slice.0,
21+
; CHECK-LABEL: test_fvdot32_bottom_1x4_indexed:
22+
; CHECK: // %bb.0:
23+
; CHECK: mov w8, w0
24+
; CHECK: fvdotb za.s[w8, 7, vgx4], { z0.b, z1.b }, z2.b[3]
25+
; CHECK: ret
26+
<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2,
27+
<vscale x 16 x i8> %zm) #0 {
28+
%slice = add i32 %slice.0, 7
29+
call void @llvm.aarch64.sme.fp8.fvdotb.lane.za32.vg1x4(i32 %slice,
30+
<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2,
31+
<vscale x 16 x i8> %zm, i32 3)
32+
ret void
33+
}
34+
35+
define void @test_fvdot32_top_1x4_indexed(i32 %slice.0,
36+
; CHECK-LABEL: test_fvdot32_top_1x4_indexed:
37+
; CHECK: // %bb.0:
38+
; CHECK: mov w8, w0
39+
; CHECK: fvdott za.s[w8, 7, vgx4], { z0.b, z1.b }, z2.b[3]
40+
; CHECK: ret
41+
<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2,
42+
<vscale x 16 x i8> %zm) #0 {
43+
%slice = add i32 %slice.0, 7
44+
call void @llvm.aarch64.sme.fp8.fvdott.lane.za32.vg1x4(i32 %slice,
45+
<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2,
46+
<vscale x 16 x i8> %zm, i32 3)
47+
ret void
48+
}
49+
50+
attributes #0 = { "target-features" = "+sme-f8f32,+sme-f8f16" }

0 commit comments

Comments
 (0)