Skip to content

Commit 7935b36

Browse files
qti-vaiskvqti-vaiskv
committed
Added SME1 support for int8 GEMM and IGEMM operations
1 parent 240fc77 commit 7935b36

22 files changed

+415
-56
lines changed

‎bench/pqs8-qc8w-gemm-minmax.cc‎

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@
3030
GEMMBenchmark(state,
3131
xnn_pqs8_qc8w_gemm_minmax_ukernel_1x32c4__neonsme2,
3232
xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params,
33-
xnn_pack_kai_qs8_qc8w_weights_and_biases_sme2,
34-
xnn_packed_stride_kai_qs8_qc8w_weights_and_biases_sme2,
33+
xnn_pack_kai_qs8_qc8w_weights_and_biases_sme,
34+
xnn_packed_stride_kai_qs8_qc8w_weights_and_biases_sme,
3535
/*mr=*/[]() -> size_t {
3636
const struct xnn_hardware_config* hardware_config =
3737
xnn_init_hardware_config();
@@ -70,8 +70,8 @@
7070
GEMMBenchmark(state,
7171
xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme2,
7272
xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params,
73-
xnn_pack_kai_qs8_qc8w_weights_and_biases_sme2,
74-
xnn_packed_stride_kai_qs8_qc8w_weights_and_biases_sme2,
73+
xnn_pack_kai_qs8_qc8w_weights_and_biases_sme,
74+
xnn_packed_stride_kai_qs8_qc8w_weights_and_biases_sme,
7575
/*mr=*/[]() -> size_t {
7676
const struct xnn_hardware_config* hardware_config =
7777
xnn_init_hardware_config();
@@ -109,6 +109,51 @@
109109
#endif // XNN_ENABLE_ARM_SME2 && XNN_ARCH_ARM64
110110

111111

112+
#if XNN_ENABLE_ARM_SME && XNN_ARCH_ARM64
113+
#if XNN_ENABLE_KLEIDIAI
114+
static void pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme(benchmark::State& state, const char* net) {
115+
GEMMBenchmark(state,
116+
xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme,
117+
xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params,
118+
xnn_pack_kai_qs8_qc8w_weights_and_biases_sme,
119+
xnn_packed_stride_kai_qs8_qc8w_weights_and_biases_sme,
120+
/*mr=*/[]() -> size_t {
121+
const struct xnn_hardware_config* hardware_config =
122+
xnn_init_hardware_config();
123+
if (hardware_config != nullptr && (hardware_config->arch_flags & xnn_arch_arm_sme) == xnn_arch_arm_sme) {
124+
return xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme_get_mr();
125+
} else {
126+
return 0;
127+
}
128+
}
129+
, /*nr=*/[]() -> size_t {
130+
const struct xnn_hardware_config* hardware_config =
131+
xnn_init_hardware_config();
132+
if (hardware_config != nullptr && (hardware_config->arch_flags & xnn_arch_arm_sme) == xnn_arch_arm_sme) {
133+
return xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme_get_nr();
134+
} else {
135+
return 0;
136+
}
137+
}
138+
, /*kr=*/4, /*sr=*/1,
139+
/*mr_packed=*/[]() -> size_t {
140+
const struct xnn_hardware_config* hardware_config =
141+
xnn_init_hardware_config();
142+
if (hardware_config != nullptr && (hardware_config->arch_flags & xnn_arch_arm_sme) == xnn_arch_arm_sme) {
143+
return xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme_get_mr();
144+
} else {
145+
return 0;
146+
}
147+
}
148+
,
149+
/*arch_flags=*/xnn_arch_arm_sme);
150+
}
151+
152+
BENCHMARK_GEMM(pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme)
153+
#endif // XNN_ENABLE_KLEIDIAI
154+
#endif // XNN_ENABLE_ARM_SME && XNN_ARCH_ARM64
155+
156+
112157
#ifndef XNNPACK_BENCHMARK_NO_MAIN
113158
XNN_BENCHMARK_MAIN();
114159
#endif

‎cmake/DownloadKleidiAI.cmake‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ ENDIF()
1818
# LINT.IfChange
1919
INCLUDE(ExternalProject)
2020
ExternalProject_Add(kleidiai
21-
URL https://github.com/ARM-software/kleidiai/archive/d7770c89632329a9914ef1a90289917597639cbe.zip
22-
URL_HASH SHA256=ecd433a4f7446f7f02a04e458989de8253f19187aa85e3b81b19e0b60f0bf859
21+
URL https://github.com/ARM-software/kleidiai/archive/bd2e6ae060014035e25bf4986be682762c446c2d.zip
22+
URL_HASH SHA256=6a4a4e16b695fd6add6c361de1ebf3c7226f954ae103bc8d71fe6705a41cfd04
2323
SOURCE_DIR "${CMAKE_BINARY_DIR}/kleidiai-source"
2424
BINARY_DIR "${CMAKE_BINARY_DIR}/kleidiai"
2525
CONFIGURE_COMMAND ""

‎cmake/gen/neonsme2_microkernels.cmake‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@ SET(PROD_NEONSME2_MICROKERNEL_SRCS
2020
src/qp8-f32-qc4w-gemm/qp8-f32-qc4w-gemm-minmax-1x64c4-neonsme2.c
2121
src/qp8-f32-qc4w-gemm/qp8-f32-qc4w-gemm-minmax-16x64c4-neonsme2.c
2222
src/qp8-f32-qc8w-gemm/qp8-f32-qc8w-gemm-minmax-1x64c4-neonsme2.c
23-
src/qp8-f32-qc8w-gemm/qp8-f32-qc8w-gemm-minmax-16x64c4-neonsme2.c
24-
src/x8-pack-lh/x8-packlh-igemm-neonsme2.c
25-
src/x8-pack-lh/x8-packlh-neonsme2.c
23+
src/qp8-f32-qc8w-gemm/qp8-f32-qc8w-gemm-minmax-16x64c4-neonsme2.c
2624
src/x16-pack-lh/x16-packlh-neonsme2.c)
2725

2826
SET(NON_PROD_NEONSME2_MICROKERNEL_SRCS)

‎cmake/gen/neonsme_microkernels.cmake‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@
1212
SET(PROD_NEONSME_MICROKERNEL_SRCS
1313
src/pf32-gemm/pf32-gemm-1x32-minmax-neonsme.c
1414
src/pf32-gemm/pf32-gemm-32x32-minmax-neonsme.c
15+
src/pqs8-qc8w-gemm/pqs8-qc8w-gemm-32x32c4-minmax-neonsme.c
16+
src/pqs8-f32-qc8w-igemm/pqs8-f32-qc8w-igemm-32x32c4-minmax-neonsme.c
17+
src/x8-pack-lh/x8-packlh-neonsme.c
18+
src/x8-pack-lh/x8-packlh-igemm-neonsme.c
1519
src/x32-pack-lh/x32-packlh-neonsme.c)
1620

1721
SET(NON_PROD_NEONSME_MICROKERNEL_SRCS)

‎gen/neonsme2_microkernels.bzl‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,7 @@ PROD_NEONSME2_MICROKERNEL_SRCS = [
1616
"src/qp8-f32-qc4w-gemm/qp8-f32-qc4w-gemm-minmax-1x64c4-neonsme2.c",
1717
"src/qp8-f32-qc4w-gemm/qp8-f32-qc4w-gemm-minmax-16x64c4-neonsme2.c",
1818
"src/qp8-f32-qc8w-gemm/qp8-f32-qc8w-gemm-minmax-1x64c4-neonsme2.c",
19-
"src/qp8-f32-qc8w-gemm/qp8-f32-qc8w-gemm-minmax-16x64c4-neonsme2.c",
20-
"src/x8-pack-lh/x8-packlh-igemm-neonsme2.c",
21-
"src/x8-pack-lh/x8-packlh-neonsme2.c",
19+
"src/qp8-f32-qc8w-gemm/qp8-f32-qc8w-gemm-minmax-16x64c4-neonsme2.c",
2220
"src/x16-pack-lh/x16-packlh-neonsme2.c",
2321
]
2422

‎gen/neonsme_microkernels.bzl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,11 @@ Auto-generated file. Do not edit!
88
PROD_NEONSME_MICROKERNEL_SRCS = [
99
"src/pf32-gemm/pf32-gemm-1x32-minmax-neonsme.c",
1010
"src/pf32-gemm/pf32-gemm-32x32-minmax-neonsme.c",
11+
"src/pqs8-f32-qc8w-igemm/pqs8-f32-qc8w-igemm-32x32c4-minmax-neonsme.c",
12+
"src/pqs8-qc8w-gemm/pqs8-qc8w-gemm-32x32c4-minmax-neonsme.c",
1113
"src/x32-pack-lh/x32-packlh-neonsme.c",
14+
"src/x8-pack-lh/x8-packlh-igemm-neonsme.c",
15+
"src/x8-pack-lh/x8-packlh-neonsme.c",
1216
]
1317

1418
NON_PROD_NEONSME_MICROKERNEL_SRCS = [

‎src/configs/gemm-config.c‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -450,11 +450,11 @@ static void init_pqs8_qc8w_gemm_config(void) {
450450
pqs8_qc8w_gemm_config.init.qs8_qc8w =
451451
xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params;
452452
pqs8_qc8w_gemm_config.pack_weights_and_biases =
453-
xnn_pack_kai_qs8_qc8w_weights_and_biases_sme2;
453+
xnn_pack_kai_qs8_qc8w_weights_and_biases_sme;
454454
pqs8_qc8w_gemm_config.packed_stride_weights_and_biases =
455-
xnn_packed_stride_kai_qs8_qc8w_weights_and_biases_sme2;
455+
xnn_packed_stride_kai_qs8_qc8w_weights_and_biases_sme;
456456
pqs8_qc8w_gemm_config.pack_igemm_goki =
457-
(xnn_pack_conv_goki_w_fn)xnn_pack_kai_qs8_conv_goki_w_sme2;
457+
(xnn_pack_conv_goki_w_fn)xnn_pack_kai_qs8_conv_goki_w_sme;
458458
pqs8_qc8w_gemm_config.pack_igemm_kgo =
459459
(xnn_pack_conv_kgo_w_fn)xnn_pack_qs8_conv_kgo_w;
460460
pqs8_qc8w_gemm_config.pack_deconv_goki =
@@ -464,6 +464,39 @@ static void init_pqs8_qc8w_gemm_config(void) {
464464
pqs8_qc8w_gemm_config.nr = nr;
465465
pqs8_qc8w_gemm_config.log2_kr = 2;
466466
#endif // XNN_ENABLE_ARM_SME2
467+
} else if (XNN_ENABLE_ARM_SME && (hardware_config->arch_flags & xnn_arch_arm_sme)) {
468+
#if XNN_ENABLE_ARM_SME
469+
const size_t mr =
470+
xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme_get_mr();
471+
const size_t nr =
472+
xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme_get_nr();
473+
pqs8_qc8w_gemm_config.arch = xnn_arch_arm_sme;
474+
pqs8_qc8w_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(mr)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme);
475+
pqs8_qc8w_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(mr)] =
476+
xnn_init_hmp_packed_igemm_ukernel(
477+
(xnn_packed_lhs_igemm_ukernel_fn)
478+
xnn_pqs8_qc8w_igemm_minmax_fp32_ukernel_32x32c4__neonsme);
479+
pqs8_qc8w_gemm_config.init.qs8_qc8w =
480+
xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params;
481+
pqs8_qc8w_gemm_config.pack_weights_and_biases =
482+
xnn_pack_kai_qs8_qc8w_weights_and_biases_sme;
483+
pqs8_qc8w_gemm_config.packed_stride_weights_and_biases =
484+
xnn_packed_stride_kai_qs8_qc8w_weights_and_biases_sme;
485+
pqs8_qc8w_gemm_config.pack_igemm_goki =
486+
(xnn_pack_conv_goki_w_fn)xnn_pack_kai_qs8_conv_goki_w_sme;
487+
pqs8_qc8w_gemm_config.pack_igemm_kgo =
488+
(xnn_pack_conv_kgo_w_fn)xnn_pack_qs8_conv_kgo_w;
489+
pqs8_qc8w_gemm_config.pack_deconv_goki =
490+
(xnn_pack_deconv_goki_w_fn)xnn_pack_qs8_deconv_goki_w;
491+
pqs8_qc8w_gemm_config.mr = mr;
492+
pqs8_qc8w_gemm_config.mr_packed = mr;
493+
pqs8_qc8w_gemm_config.nr = nr;
494+
pqs8_qc8w_gemm_config.log2_kr = 2;
495+
#endif // XNN_ENABLE_ARM_SME
496+
497+
}
498+
else {
499+
/* No action */
467500
}
468501
assert(pqs8_qc8w_gemm_config.mr <= XNN_MAX_MR);
469502
#endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI

‎src/configs/pack-lh-config.c‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -103,13 +103,13 @@ const struct xnn_pack_lh_config* xnn_init_x16_pack_lh_config() {
103103

104104
static void init_x8_pack_lh_config(void) {
105105
#if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI
106-
#if XNN_ENABLE_ARM_SME2
106+
#if XNN_ENABLE_ARM_SME
107107
const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();
108108
assert(hardware_config != NULL);
109-
if (hardware_config->arch_flags & xnn_arch_arm_sme2) {
110-
x8_pack_lh_config.pack_lh_fn = (xnn_pack_lh_ukernel_fn) xnn_x8_pack_lh_ukernel__neonsme2;
111-
x8_pack_lh_config.size_fn = (xnn_pack_lh_size_fn) xnn_x8_pack_lh_size__neonsme2;
112-
x8_pack_lh_config.offset_fn = (xnn_pack_lh_offset_fn) xnn_x8_pack_lh_offset__neonsme2;
109+
if ((hardware_config->arch_flags & xnn_arch_arm_sme)) {
110+
x8_pack_lh_config.pack_lh_fn = (xnn_pack_lh_ukernel_fn) xnn_x8_pack_lh_ukernel__neonsme;
111+
x8_pack_lh_config.size_fn = (xnn_pack_lh_size_fn) xnn_x8_pack_lh_size__neonsme;
112+
x8_pack_lh_config.offset_fn = (xnn_pack_lh_offset_fn) xnn_x8_pack_lh_offset__neonsme;
113113
}
114114
#endif // XNN_ENABLE_ARM_SME2
115115
#endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI
@@ -129,13 +129,13 @@ const struct xnn_pack_lh_config* xnn_init_x8_pack_lh_config() {
129129

130130
static void init_x8_igemm_pack_lh_config(void) {
131131
#if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI
132-
#if XNN_ENABLE_ARM_SME2
132+
#if XNN_ENABLE_ARM_SME
133133
const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();
134134
assert(hardware_config != NULL);
135-
if (hardware_config->arch_flags & xnn_arch_arm_sme2) {
136-
x8_igemm_pack_lh_config.pack_lh_for_igemm_fn = (xnn_pack_lh_igemm_ukernel_fn) xnn_x8_pack_lh_ukernel__igemm_neonsme2;
137-
x8_igemm_pack_lh_config.size_for_igemm_fn = (xnn_pack_lh_igemm_size_fn) xnn_x8_pack_lh_size__igemm_neonsme2;
138-
x8_igemm_pack_lh_config.offset_for_igemm_fn = (xnn_pack_lh_igemm_offset_fn) xnn_x8_pack_lh_offset__igemm_neonsme2;
135+
if ((hardware_config->arch_flags & xnn_arch_arm_sme)) {
136+
x8_igemm_pack_lh_config.pack_lh_for_igemm_fn = (xnn_pack_lh_igemm_ukernel_fn) xnn_x8_pack_lh_ukernel__igemm_neonsme;
137+
x8_igemm_pack_lh_config.size_for_igemm_fn = (xnn_pack_lh_igemm_size_fn) xnn_x8_pack_lh_size__igemm_neonsme;
138+
x8_igemm_pack_lh_config.offset_for_igemm_fn = (xnn_pack_lh_igemm_offset_fn) xnn_x8_pack_lh_offset__igemm_neonsme;
139139
}
140140
#endif // XNN_ENABLE_ARM_SME2
141141
#endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI

‎src/pqs8-f32-qc8w-igemm/pqs8-f32-qc8w-igemm-32x32c4-minmax-neonsme.c‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// This source code is licensed under the BSD-style license found in the
4+
// LICENSE file in the root directory of this source tree.
5+
6+
#include <stddef.h>
7+
8+
#include "src/xnnpack/microparams.h"
9+
10+
#if XNN_ENABLE_KLEIDIAI
11+
#include "kai/ukernels/matmul/matmul_clamp_qai8_qai8p_qsi8cxp/kai_matmul_clamp_qai8_qai8p2vlx4_qsi8cxp2vlx4sb_2vlx2vl_sme_mopa.h"
12+
#include "kai/ukernels/matmul/pack/kai_lhs_imatmul_pack_x8p2vlx4_x8p_sme.h"
13+
#endif // XNN_ENABLE_KLEIDIAI
14+
15+
size_t xnn_pqs8_qc8w_igemm_minmax_fp32_ukernel_32x32c4__neonsme_get_mr(void) {
16+
#if XNN_ENABLE_KLEIDIAI
17+
return kai_get_mr_matmul_clamp_qai8_qai8p2vlx4_qsi8cxp2vlx4sb_2vlx2vl_sme_mopa();
18+
#else
19+
assert(
20+
"Calling wrapped KleidiAI function, but XNNPACK was compiled without "
21+
"`XNN_ENABLE_KLEIDIAI`." &&
22+
0);
23+
return 0;
24+
#endif // XNN_ENABLE_KLEIDIAI
25+
}
26+
27+
size_t xnn_pqs8_qc8w_igemm_minmax_fp32_ukernel_32x32c4__neonsme_get_nr(void) {
28+
#if XNN_ENABLE_KLEIDIAI
29+
return kai_get_nr_matmul_clamp_qai8_qai8p2vlx4_qsi8cxp2vlx4sb_2vlx2vl_sme_mopa();
30+
#else
31+
assert(
32+
"Calling wrapped KleidiAI function, but XNNPACK was compiled without "
33+
"`XNN_ENABLE_KLEIDIAI`." &&
34+
0);
35+
return 0;
36+
#endif // XNN_ENABLE_KLEIDIAI
37+
}
38+
39+
void xnn_pqs8_qc8w_igemm_minmax_fp32_ukernel_32x32c4__neonsme(
40+
size_t mr, size_t nc, size_t kc, size_t ks, const void* packed_lhs,
41+
const void* restrict w, int8_t* restrict c, size_t cm_stride,
42+
const union xnn_qs8_qc8w_conv_minmax_params* params) {
43+
#if XNN_ENABLE_KLEIDIAI
44+
const size_t kai_kr = 4;
45+
const size_t k = ks * round_up(kc, kai_kr);
46+
47+
// Repackage the params.
48+
struct kai_matmul_requantize32_params kai_params;
49+
kai_params.output_zero_point = params->fp32_scalar.output_zero_point;
50+
kai_params.min_value = (int8_t)params->fp32_scalar.output_min;
51+
kai_params.max_value = (int8_t)params->fp32_scalar.output_max;
52+
53+
kai_run_matmul_clamp_qai8_qai8p2vlx4_qsi8cxp2vlx4sb_2vlx2vl_sme_mopa(
54+
mr, nc, k, packed_lhs, w, c, cm_stride, sizeof(int8_t), &kai_params);
55+
#else
56+
assert(
57+
"Calling wrapped KleidiAI function, but XNNPACK was compiled without "
58+
"`XNN_ENABLE_KLEIDIAI`." &&
59+
0);
60+
#endif // XNN_ENABLE_KLEIDIAI
61+
}

‎src/pqs8-qc8w-gemm/pqs8-qc8w-gemm-32x32c4-minmax-neonsme.c‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// This source code is licensed under the BSD-style license found in the
4+
// LICENSE file in the root directory of this source tree.
5+
6+
#include <stddef.h>
7+
8+
#include "src/xnnpack/math.h"
9+
#include "src/xnnpack/microparams.h"
10+
11+
#if XNN_ENABLE_KLEIDIAI
12+
#include "kai/ukernels/matmul/matmul_clamp_qai8_qai8p_qsi8cxp/kai_matmul_clamp_qai8_qai8p2vlx4_qsi8cxp2vlx4sb_2vlx2vl_sme_mopa.h"
13+
#endif // XNN_ENABLE_KLEIDIAI
14+
15+
16+
size_t xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme_get_mr() {
17+
#if XNN_ENABLE_KLEIDIAI
18+
return kai_get_mr_matmul_clamp_qai8_qai8p2vlx4_qsi8cxp2vlx4sb_2vlx2vl_sme_mopa();
19+
#else
20+
assert(
21+
"Calling KleidiAI kai_get_mr wrapper, but XNNPACK was compiled without "
22+
"`XNN_ENABLE_KLEIDIAI`." && 0);
23+
return 0;
24+
#endif // XNN_ENABLE_KLEIDIAI
25+
}
26+
27+
size_t xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme_get_nr() {
28+
#if XNN_ENABLE_KLEIDIAI
29+
return kai_get_nr_matmul_clamp_qai8_qai8p2vlx4_qsi8cxp2vlx4sb_2vlx2vl_sme_mopa();
30+
31+
#else
32+
assert(
33+
"Calling KleidiAI kai_get_nr wrapper, but XNNPACK was compiled without "
34+
"`XNN_ENABLE_KLEIDIAI`." && 0);
35+
return 0;
36+
#endif // XNN_ENABLE_KLEIDIAI
37+
}
38+
39+
// Wraps the `kai_run_matmul_clamp_qai8_qai8p2vlx4_qsi8cxpsb2vlx4_2vlx2vl_sme_mopa`
40+
// GEMM microkernel with a name that is compatible with our tooling.
41+
void xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme(
42+
size_t m, size_t n, size_t k, const void* lhs_packed,
43+
const void* rhs_packed, void* dst, size_t dst_stride_row,
44+
size_t dst_stride_col,
45+
const union xnn_qs8_qc8w_conv_minmax_params* minmax_params) {
46+
#if XNN_ENABLE_KLEIDIAI
47+
struct kai_matmul_requantize32_params kai_params;
48+
kai_params.output_zero_point = minmax_params->fp32_scalar.output_zero_point;
49+
kai_params.min_value = minmax_params->fp32_scalar.output_min;
50+
kai_params.max_value = minmax_params->fp32_scalar.output_max;
51+
52+
kai_run_matmul_clamp_qai8_qai8p2vlx4_qsi8cxp2vlx4sb_2vlx2vl_sme_mopa(
53+
m, n, k / sizeof(int8_t), lhs_packed, rhs_packed, dst, dst_stride_row,
54+
/*dst_stride_col=*/sizeof(int8_t), &kai_params);
55+
#else
56+
assert(
57+
"Calling KleidiAI microkernel wrapper, but XNNPACK was compiled without "
58+
"`XNN_ENABLE_KLEIDIAI`." && 0);
59+
#endif // XNN_ENABLE_KLEIDIAI
60+
}

0 commit comments

Comments
 (0)