From 5bf523234bcfc12c284226a40303f0bafc128f98 Mon Sep 17 00:00:00 2001 From: Quentin Khan Date: Thu, 11 Sep 2025 10:35:28 -0700 Subject: [PATCH] Use the new fingerprinting method for XNNPack delegate cache files. Warning: this new fingerprinting method is currently experimental. We now version the micro-kernel configuration for operations that make use of the weight cache. This allows the cache provider to keep track of these versions to check, when restoring packed weights from a previous run, that they were packed in a way that is compatible with the current XNNPack version. PiperOrigin-RevId: 805897120 --- include/experimental.h | 17 ++++ include/xnnpack.h | 5 + src/configs/BUILD | 1 + src/configs/config-identifier.c | 79 ++++++++++++++++ src/configs/gemm-config.c | 145 +++++++++++++++++++++++++++++ src/operators/fully-connected-nc.c | 1 + src/xnnpack/config-types.h | 2 + src/xnnpack/config.h | 59 ++++++++++-- test/BUILD.bazel | 9 ++ test/config.cc | 22 +++++ 10 files changed, 333 insertions(+), 7 deletions(-) create mode 100644 src/configs/config-identifier.c create mode 100644 test/config.cc diff --git a/include/experimental.h b/include/experimental.h index be576d8ad41..f165ca1a07a 100644 --- a/include/experimental.h +++ b/include/experimental.h @@ -96,6 +96,23 @@ enum xnn_status xnn_update_runtime_with_threadpool( xnn_runtime_t runtime, xnn_threadpool_t threadpool); + +typedef struct xnn_config_identifier { + uint64_t identifier; +} xnn_config_identifier; + +/// Check whether the given configuration matches one that is currently in use. +/// +/// @returns True if the configuration matches. +bool xnn_check_config_version(const struct xnn_config_identifier* identifier); + +/// Returns a valid microkernel configuration. +/// +/// This is useful for consumers of this API that want to test their checks +/// against kernel configurations. +const struct xnn_config_identifier* xnn_get_test_config(); + + #ifdef __cplusplus } // extern "C" #endif diff --git a/include/xnnpack.h b/include/xnnpack.h index 6750135adb7..8aa3e050868 100644 --- a/include/xnnpack.h +++ b/include/xnnpack.h @@ -2294,6 +2294,11 @@ struct xnn_weights_cache_look_up_key { const void* kernel; /// Pointer to the original bias, could be NULL. const void* bias; + /// Pointer to the operation configuration, can be NULL. + /// + /// If this is set, then the cache is allowed to compare the configuration to + /// previous runs and maybe reuse those run results. + const struct xnn_config_identifier* config; }; /// A group of function pointers to manage weights cache. All functions may be diff --git a/src/configs/BUILD b/src/configs/BUILD index 2552a025604..bdab781ea31 100644 --- a/src/configs/BUILD +++ b/src/configs/BUILD @@ -69,6 +69,7 @@ xnnpack_cc_library( "avgpool-config.c", "binary-elementwise-config.c", "cmul-config.c", + "config-identifier.c", "conv-hwc2chw-config.c", "dwconv-config.c", "dwconv2d-chw-config.c", diff --git a/src/configs/config-identifier.c b/src/configs/config-identifier.c new file mode 100644 index 00000000000..720c6e1fb42 --- /dev/null +++ b/src/configs/config-identifier.c @@ -0,0 +1,79 @@ +// Copyright 2025 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include +#include +#include + +#include "include/experimental.h" +#include "src/xnnpack/config-types.h" +#include "src/xnnpack/config.h" + +xnn_config_identifier xnn_create_config_identifier(xnn_config_name name, + uint32_t version) { + struct xnn_config_identifier id = {((uint64_t)name) << 32 | version}; + return id; +} + +xnn_config_name xnn_get_config_name(const xnn_config_identifier* identifier) { + return identifier->identifier >> 32; +} + +xnn_config_name xnn_get_config_version( + const xnn_config_identifier* identifier) { + return identifier->identifier & 0xffffffff; +} + +const xnn_config_identifier* xnn_get_test_config() { + const struct xnn_gemm_config* f32_config = + xnn_init_f32_gemm_config(/*flags=*/0); + return &(f32_config->identifier); +} + +#define XNNPACK_CHECK_CONFIG(CONFIG_NAME, ...) \ + if (config_name == xnn_config_name_##CONFIG_NAME) { \ + const struct xnn_gemm_config* kernel_config = \ + xnn_init_##CONFIG_NAME##_config(__VA_ARGS__); \ + return kernel_config && \ + identifier->identifier == kernel_config->identifier.identifier; \ + } + +bool xnn_check_config_version(const xnn_config_identifier* identifier) { + if (identifier == NULL) { + return false; + } + const xnn_config_name config_name = xnn_get_config_name(identifier); + XNNPACK_CHECK_CONFIG(bf16_f32_gemm); + XNNPACK_CHECK_CONFIG(f16_gemm); + XNNPACK_CHECK_CONFIG(f32_gemm, /*flags=*/0); + XNNPACK_CHECK_CONFIG(f32_gemm_nr2, /*flags=*/0); + XNNPACK_CHECK_CONFIG(f32_igemm); + XNNPACK_CHECK_CONFIG(f32_qc8w_gemm); + XNNPACK_CHECK_CONFIG(f32_qc4w_gemm); + XNNPACK_CHECK_CONFIG(pf16_gemm); + XNNPACK_CHECK_CONFIG(pf32_gemm); + XNNPACK_CHECK_CONFIG(pqs8_qc8w_gemm); + XNNPACK_CHECK_CONFIG(qd8_f16_qb4w_gemm); + XNNPACK_CHECK_CONFIG(qd8_f16_qc4w_gemm); + XNNPACK_CHECK_CONFIG(qd8_f16_qc8w_gemm); + XNNPACK_CHECK_CONFIG(qd8_f16_qc8w_igemm); + XNNPACK_CHECK_CONFIG(qd8_f32_qb4w_gemm); + XNNPACK_CHECK_CONFIG(qd8_f32_qc4w_gemm); + XNNPACK_CHECK_CONFIG(qd8_f32_qc8w_gemm); + XNNPACK_CHECK_CONFIG(qp8_f32_qc4w_gemm); + XNNPACK_CHECK_CONFIG(qp8_f32_qc8w_gemm); + XNNPACK_CHECK_CONFIG(qp8_f32_qb4w_gemm); + XNNPACK_CHECK_CONFIG(qdu8_f32_qc4w_gemm); + XNNPACK_CHECK_CONFIG(qdu8_f16_qc8w_gemm); + XNNPACK_CHECK_CONFIG(qdu8_f32_qc8w_gemm); + XNNPACK_CHECK_CONFIG(qdu8_f32_qb4w_gemm); + XNNPACK_CHECK_CONFIG(qdu8_f16_qc4w_gemm); + XNNPACK_CHECK_CONFIG(qdu8_f32_qc8w_igemm); + XNNPACK_CHECK_CONFIG(qs8_qc4w_gemm); + XNNPACK_CHECK_CONFIG(qs8_qc8w_gemm); + XNNPACK_CHECK_CONFIG(qu8_gemm); + return false; +} diff --git a/src/configs/gemm-config.c b/src/configs/gemm-config.c index 3f2e5bbb281..61d2178e5ea 100644 --- a/src/configs/gemm-config.c +++ b/src/configs/gemm-config.c @@ -137,6 +137,10 @@ XNN_INIT_ONCE_GUARD(qu8_gemm); xnn_log_info("Using qp8gemm_bl microkernel '%s'.", #ukernel); static void init_f16_gemm_config(void) { + // LINT.IfChange(init_f16_gemm_config_identifier) + f16_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_f16_gemm, /*version=*/0); + // LINT.ThenChange(:init_f16_gemm_config_config) + // LINT.IfChange(init_f16_gemm_config_config) #if XNN_ARCH_ARM && XNN_ENABLE_ARM_FP16_VECTOR && XNN_ENABLE_ARM_FP16_SCALAR const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); assert(hardware_config != NULL); @@ -287,6 +291,7 @@ static void init_f16_gemm_config(void) { } #endif assert(f16_gemm_config.mr <= XNN_MAX_MR); + // LINT.ThenChange(:init_f16_gemm_config_identifier) } #if XNN_ARCH_WASMSIMD @@ -305,6 +310,10 @@ static void init_f16_gemm_config(void) { #endif static void init_pf16_gemm_config(void) { + // LINT.IfChange(init_pf16_gemm_config_identifier) + pf16_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_pf16_gemm, /*version=*/0); + // LINT.ThenChange(:init_pf16_gemm_config_config) + // LINT.IfChange(init_pf16_gemm_config_config) #if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); @@ -326,9 +335,14 @@ static void init_pf16_gemm_config(void) { #endif // XNN_ENABLE_ARM_SME2 } #endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI + // LINT.ThenChange(:init_pf16_gemm_config_identifier) } static void init_bf16_f32_gemm_config(void) { + // LINT.IfChange(init_bf16_f32_gemm_config_identifier) + bf16_f32_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_bf16_f32_gemm, /*version=*/0); + // LINT.ThenChange(:init_bf16_f32_gemm_config_config) + // LINT.IfChange(init_bf16_f32_gemm_config_config) #if XNN_ARCH_X86_64 const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); assert(hardware_config != NULL); @@ -347,9 +361,14 @@ static void init_bf16_f32_gemm_config(void) { } assert(bf16_f32_gemm_config.mr <= XNN_MAX_MR); #endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI + // LINT.ThenChange(:init_bf16_f32_gemm_config_identifier) } static void init_pf32_gemm_config(void) { + // LINT.IfChange(init_pf32_gemm_config_identifier) + pf32_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_pf32_gemm, /*version=*/0); + // LINT.ThenChange(:init_pf32_gemm_config_config) + // LINT.IfChange(init_pf32_gemm_config_config) #if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); assert(hardware_config != NULL); @@ -388,9 +407,14 @@ static void init_pf32_gemm_config(void) { } assert(pf32_gemm_config.mr <= XNN_MAX_MR); #endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI + // LINT.ThenChange(:init_pf32_gemm_config_identifier) } static void init_pqs8_qc8w_gemm_config(void) { + // LINT.IfChange(init_pqs8_qc8w_gemm_config_identifier) + pqs8_qc8w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_pqs8_qc8w_gemm, /*version=*/0); + // LINT.ThenChange(:init_pqs8_qc8w_gemm_config_config) + // LINT.IfChange(init_pqs8_qc8w_gemm_config_config) #if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); @@ -429,9 +453,14 @@ static void init_pqs8_qc8w_gemm_config(void) { } assert(pqs8_qc8w_gemm_config.mr <= XNN_MAX_MR); #endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI + // LINT.ThenChange(:init_pqs8_qc8w_gemm_config_identifier) } static void init_f32_gemm_config_impl(struct xnn_gemm_config* f32_gemm_config, bool consistent_arithmetic) { + // LINT.IfChange(init_f32_gemm_config_identifier) + f32_gemm_config->identifier = xnn_create_config_identifier(xnn_config_name_f32_gemm, /*version=*/0); + // LINT.ThenChange(:init_f32_gemm_config_config) + // LINT.IfChange(init_f32_gemm_config_config) #if XNN_ARCH_ARM const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); assert(hardware_config != NULL); @@ -929,6 +958,7 @@ static void init_f32_gemm_config_impl(struct xnn_gemm_config* f32_gemm_config, b f32_gemm_config->nr = 4; #endif assert(f32_gemm_config->mr <= XNN_MAX_MR); + // LINT.ThenChange(:init_f32_gemm_config_identifier) } static void init_f32_gemm_config() { @@ -937,6 +967,10 @@ static void init_f32_gemm_config() { } static void init_f32_igemm_config(void) { + // LINT.IfChange(init_f32_igemm_config_identifier) + f32_igemm_config.identifier = xnn_create_config_identifier(xnn_config_name_f32_igemm, /*version=*/0); + // LINT.ThenChange(:init_f32_igemm_config_config) + // LINT.IfChange(init_f32_igemm_config_config) #if XNN_ARCH_ARM const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); assert(hardware_config != NULL); @@ -1409,9 +1443,14 @@ static void init_f32_igemm_config(void) { f32_igemm_config.nr = 4; #endif assert(f32_igemm_config.mr <= XNN_MAX_MR); + // LINT.ThenChange(:init_f32_igemm_config_identifier) } static void init_f32_gemm_nr2_config_impl(struct xnn_gemm_config* f32_gemm_nr2_config, bool consistent_arithmetic) { + // LINT.IfChange(init_f32_gemm_nr2_config_identifier) + f32_gemm_nr2_config->identifier = xnn_create_config_identifier(xnn_config_name_f32_gemm_nr2, /*version=*/0); + // LINT.ThenChange(:init_f32_gemm_nr2_config_config) + // LINT.IfChange(init_f32_gemm_nr2_config_config) #if XNN_ARCH_ARM const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); assert(hardware_config != NULL); @@ -1605,6 +1644,7 @@ static void init_f32_gemm_nr2_config_impl(struct xnn_gemm_config* f32_gemm_nr2_c f32_gemm_nr2_config->nr = 2; #endif assert(f32_gemm_nr2_config->mr <= XNN_MAX_MR); + // LINT.ThenChange(:init_f32_gemm_nr2_config_identifier) } static void init_f32_gemm_nr2_config() { @@ -1613,6 +1653,10 @@ static void init_f32_gemm_nr2_config() { } static void init_f32_qc4w_gemm_config(void) { + // LINT.IfChange(init_f32_qc4w_gemm_config_identifier) + f32_qc4w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_f32_qc4w_gemm, /*version=*/0); + // LINT.ThenChange(:init_f32_qc4w_gemm_config_config) + // LINT.IfChange(init_f32_qc4w_gemm_config_config) f32_qc4w_gemm_config.planes = 1; #if XNN_ARCH_ARM const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); @@ -1693,9 +1737,14 @@ static void init_f32_qc4w_gemm_config(void) { f32_qc4w_gemm_config.nr = 4; #endif assert(f32_qc4w_gemm_config.mr <= XNN_MAX_MR); + // LINT.ThenChange(:init_f32_qc4w_gemm_config_identifier) } static void init_f32_qc8w_gemm_config(void) { + // LINT.IfChange(init_f32_qc8w_gemm_config_identifier) + f32_qc8w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_f32_qc8w_gemm, /*version=*/0); + // LINT.ThenChange(:init_f32_qc8w_gemm_config_config) + // LINT.IfChange(init_f32_qc8w_gemm_config_config) #if XNN_ARCH_ARM const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); assert(hardware_config != NULL); @@ -1901,9 +1950,14 @@ static void init_f32_qc8w_gemm_config(void) { f32_qc8w_gemm_config.nr = 4; #endif assert(f32_qc8w_gemm_config.mr <= XNN_MAX_MR); + // LINT.ThenChange(:init_f32_qc8w_gemm_config_identifier) } static void init_qdu8_f16_qc4w_gemm_config(void) { + // LINT.IfChange(init_qdu8_f16_qc4w_gemm_config_identifier) + qdu8_f16_qc4w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qdu8_f16_qc4w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qdu8_f16_qc4w_gemm_config_config) + // LINT.IfChange(init_qdu8_f16_qc4w_gemm_config_config) // Use the same packing function throughout. qdu8_f16_qc4w_gemm_config.pack_weights_and_biases = (xnn_pack_weights_and_biases_fn)xnn_pack_qs4_weights_and_biases; @@ -1973,9 +2027,14 @@ static void init_qdu8_f16_qc4w_gemm_config(void) { #endif assert(qdu8_f16_qc4w_gemm_config.mr <= XNN_MAX_MR); assert(qdu8_f16_qc4w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); + // LINT.ThenChange(:init_qdu8_f16_qc4w_gemm_config_identifier) } static void init_qd8_f16_qc4w_gemm_config(void) { + // LINT.IfChange(init_qd8_f16_qc4w_gemm_config_identifier) + qd8_f16_qc4w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qd8_f16_qc4w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qd8_f16_qc4w_gemm_config_config) + // LINT.IfChange(init_qd8_f16_qc4w_gemm_config_config) // Use the same packing function throughout. qd8_f16_qc4w_gemm_config.pack_weights_and_biases = (xnn_pack_weights_and_biases_fn)xnn_pack_qs4_weights_and_biases; @@ -2043,9 +2102,14 @@ static void init_qd8_f16_qc4w_gemm_config(void) { #endif assert(qd8_f16_qc4w_gemm_config.mr <= XNN_MAX_MR); assert(qd8_f16_qc4w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); + // LINT.ThenChange(:init_qd8_f16_qc4w_gemm_config_identifier) } static void init_qd8_f16_qb4w_gemm_config(void) { + // LINT.IfChange(init_qd8_f16_qb4w_gemm_config_identifier) + qd8_f16_qb4w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qd8_f16_qb4w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qd8_f16_qb4w_gemm_config_config) + // LINT.IfChange(init_qd8_f16_qb4w_gemm_config_config) qd8_f16_qb4w_gemm_config.packed_stride_weights_and_biases = xnn_packed_stride_qb4_weights_and_biases; qd8_f16_qb4w_gemm_config.pack_weights_and_biases = xnn_pack_qb4_weights_and_biases; @@ -2121,9 +2185,14 @@ static void init_qd8_f16_qb4w_gemm_config(void) { #endif assert(qd8_f16_qb4w_gemm_config.mr <= XNN_MAX_MR); assert(qd8_f16_qb4w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); + // LINT.ThenChange(:init_qd8_f16_qb4w_gemm_config_identifier) } static void init_qd8_f32_qc4w_gemm_config(void) { + // LINT.IfChange(init_qd8_f32_qc4w_gemm_config_identifier) + qd8_f32_qc4w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qd8_f32_qc4w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qd8_f32_qc4w_gemm_config_config) + // LINT.IfChange(init_qd8_f32_qc4w_gemm_config_config) // Use the same packing function throughout. qd8_f32_qc4w_gemm_config.pack_weights_and_biases = (xnn_pack_weights_and_biases_fn) xnn_pack_qs4_weights_and_biases; qd8_f32_qc4w_gemm_config.packed_stride_weights_and_biases = (xnn_packed_stride_weights_and_biases_fn) xnn_packed_stride_qs4_weights_and_biases; @@ -2269,9 +2338,14 @@ static void init_qd8_f32_qc4w_gemm_config(void) { #endif assert(qd8_f32_qc4w_gemm_config.mr <= XNN_MAX_MR); assert(qd8_f32_qc4w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); + // LINT.ThenChange(:init_qd8_f32_qc4w_gemm_config_identifier) } static void init_qp8_f32_qc4w_gemm_config(void) { + // LINT.IfChange(init_qp8_f32_qc4w_gemm_config_identifier) + qp8_f32_qc4w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qp8_f32_qc4w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qp8_f32_qc4w_gemm_config_config) + // LINT.IfChange(init_qp8_f32_qc4w_gemm_config_config) #if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); @@ -2324,9 +2398,14 @@ static void init_qp8_f32_qc4w_gemm_config(void) { } assert(qp8_f32_qc4w_gemm_config.mr <= XNN_MAX_MR); #endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI + // LINT.ThenChange(:init_qp8_f32_qc4w_gemm_config_identifier) } static void init_qp8_f32_qc8w_gemm_config(void) { + // LINT.IfChange(init_qp8_f32_qc8w_gemm_config_identifier) + qp8_f32_qc8w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qp8_f32_qc8w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qp8_f32_qc8w_gemm_config_config) + // LINT.IfChange(init_qp8_f32_qc8w_gemm_config_config) #if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); @@ -2372,9 +2451,14 @@ static void init_qp8_f32_qc8w_gemm_config(void) { } assert(qp8_f32_qc8w_gemm_config.mr <= XNN_MAX_MR); #endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI + // LINT.ThenChange(:init_qp8_f32_qc8w_gemm_config_identifier) } static void init_qp8_f32_qb4w_gemm_config(void) { + // LINT.IfChange(init_qp8_f32_qb4w_gemm_config_identifier) + qp8_f32_qb4w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qp8_f32_qb4w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qp8_f32_qb4w_gemm_config_config) + // LINT.IfChange(init_qp8_f32_qb4w_gemm_config_config) #if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); @@ -2410,9 +2494,14 @@ static void init_qp8_f32_qb4w_gemm_config(void) { } assert(qp8_f32_qb4w_gemm_config.mr <= XNN_MAX_MR); #endif // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI + // LINT.ThenChange(:init_qp8_f32_qb4w_gemm_config_identifier) } static void init_qdu8_f32_qb4w_gemm_config(void) { + // LINT.IfChange(init_qdu8_f32_qb4w_gemm_config_identifier) + qdu8_f32_qb4w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qdu8_f32_qb4w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qdu8_f32_qb4w_gemm_config_config) + // LINT.IfChange(init_qdu8_f32_qb4w_gemm_config_config) qdu8_f32_qb4w_gemm_config.packed_stride_weights_and_biases = xnn_packed_stride_qb4_weights_and_biases; qdu8_f32_qb4w_gemm_config.pack_weights_and_biases = xnn_pack_qb4_weights_and_biases; #if XNN_ARCH_X86 || XNN_ARCH_X86_64 @@ -2450,9 +2539,14 @@ static void init_qdu8_f32_qb4w_gemm_config(void) { #endif assert(qdu8_f32_qb4w_gemm_config.mr <= XNN_MAX_MR); #endif // XNN_ARCH_X86 || XNN_ARCH_X86_64 + // LINT.ThenChange(:init_qdu8_f32_qb4w_gemm_config_identifier) } static void init_qd8_f32_qb4w_gemm_config(void) { + // LINT.IfChange(init_qd8_f32_qb4w_gemm_config_identifier) + qd8_f32_qb4w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qd8_f32_qb4w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qd8_f32_qb4w_gemm_config_config) + // LINT.IfChange(init_qd8_f32_qb4w_gemm_config_config) qd8_f32_qb4w_gemm_config.packed_stride_weights_and_biases = xnn_packed_stride_qb4_weights_and_biases; qd8_f32_qb4w_gemm_config.pack_weights_and_biases = xnn_pack_qb4_weights_and_biases; qd8_f32_qb4w_gemm_config.pack_gemm_goi_bl = NULL; @@ -2572,9 +2666,14 @@ static void init_qd8_f32_qb4w_gemm_config(void) { #endif assert(qd8_f32_qb4w_gemm_config.mr <= XNN_MAX_MR); assert(qd8_f32_qb4w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); + // LINT.ThenChange(:init_qd8_f32_qb4w_gemm_config_identifier) } static void init_qd8_f16_qc8w_gemm_config(void) { + // LINT.IfChange(init_qd8_f16_qc8w_gemm_config_identifier) + qd8_f16_qc8w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qd8_f16_qc8w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qd8_f16_qc8w_gemm_config_config) + // LINT.IfChange(init_qd8_f16_qc8w_gemm_config_config) // Use the same packing function throughout. qd8_f16_qc8w_gemm_config.pack_weights_and_biases = (xnn_pack_weights_and_biases_fn)xnn_pack_qs8_weights_and_biases; qd8_f16_qc8w_gemm_config.packed_stride_weights_and_biases = (xnn_packed_stride_weights_and_biases_fn) xnn_packed_stride_qs8_weights_and_biases; @@ -2858,9 +2957,14 @@ static void init_qd8_f16_qc8w_gemm_config(void) { #endif assert(qd8_f16_qc8w_gemm_config.mr <= XNN_MAX_MR); assert(qd8_f16_qc8w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); + // LINT.ThenChange(:init_qd8_f16_qc8w_gemm_config_identifier) } static void init_qdu8_f16_qc8w_gemm_config(void) { + // LINT.IfChange(init_qdu8_f16_qc8w_gemm_config_identifier) + qdu8_f16_qc8w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qdu8_f16_qc8w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qdu8_f16_qc8w_gemm_config_config) + // LINT.IfChange(init_qdu8_f16_qc8w_gemm_config_config) // Use the same packing function throughout. qdu8_f16_qc8w_gemm_config.pack_weights_and_biases = (xnn_pack_weights_and_biases_fn)xnn_pack_qs8_weights_and_biases; @@ -2915,9 +3019,14 @@ static void init_qdu8_f16_qc8w_gemm_config(void) { assert(qdu8_f16_qc8w_gemm_config.mr <= XNN_MAX_MR); assert(qdu8_f16_qc8w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); #endif // XNN_ARCH_X86 || XNN_ARCH_X86_64 + // LINT.ThenChange(:init_qdu8_f16_qc8w_gemm_config_identifier) } static void init_qd8_f16_qc8w_igemm_config(void) { + // LINT.IfChange(init_qd8_f16_qc8w_igemm_config_identifier) + qd8_f16_qc8w_igemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qd8_f16_qc8w_igemm, /*version=*/0); + // LINT.ThenChange(:init_qd8_f16_qc8w_igemm_config_config) + // LINT.IfChange(init_qd8_f16_qc8w_igemm_config_config) // Use the same packing function throughout. qd8_f16_qc8w_igemm_config.pack_weights_and_biases = (xnn_pack_weights_and_biases_fn)xnn_pack_qs8_weights_and_biases; qd8_f16_qc8w_igemm_config.packed_stride_weights_and_biases = (xnn_packed_stride_weights_and_biases_fn) xnn_packed_stride_qs8_weights_and_biases; @@ -3189,9 +3298,14 @@ static void init_qd8_f16_qc8w_igemm_config(void) { #endif assert(qd8_f16_qc8w_igemm_config.mr <= XNN_MAX_MR); assert(qd8_f16_qc8w_igemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); + // LINT.ThenChange(:init_qd8_f16_qc8w_igemm_config_identifier) } static void init_qdu8_f32_qc8w_gemm_config(void) { + // LINT.IfChange(init_qdu8_f32_qc8w_gemm_config_identifier) + qdu8_f32_qc8w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qdu8_f32_qc8w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qdu8_f32_qc8w_gemm_config_config) + // LINT.IfChange(init_qdu8_f32_qc8w_gemm_config_config) // Use the same packing function throughout. qdu8_f32_qc8w_gemm_config.pack_weights_and_biases = (xnn_pack_weights_and_biases_fn)xnn_pack_qs8_weights_and_biases; @@ -3260,9 +3374,14 @@ static void init_qdu8_f32_qc8w_gemm_config(void) { assert(qdu8_f32_qc8w_gemm_config.mr <= XNN_MAX_MR); assert(qdu8_f32_qc8w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); #endif //XNN_ARCH_X86 || XNN_ARCH_X86_64 + // LINT.ThenChange(:init_qdu8_f32_qc8w_gemm_config_identifier) } static void init_qdu8_f32_qc8w_igemm_config(void) { + // LINT.IfChange(init_qdu8_f32_qc8w_igemm_config_identifier) + qdu8_f32_qc8w_igemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qdu8_f32_qc8w_igemm, /*version=*/0); + // LINT.ThenChange(:init_qdu8_f32_qc8w_igemm_config_config) + // LINT.IfChange(init_qdu8_f32_qc8w_igemm_config_config) // Use the same packing function throughout. qdu8_f32_qc8w_igemm_config.pack_weights_and_biases = (xnn_pack_weights_and_biases_fn)xnn_pack_qs8_weights_and_biases; @@ -3316,9 +3435,14 @@ static void init_qdu8_f32_qc8w_igemm_config(void) { assert(qdu8_f32_qc8w_igemm_config.mr <= XNN_MAX_MR); assert(qdu8_f32_qc8w_igemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); #endif //XNN_ARCH_X86 || XNN_ARCH_X86_64 + // LINT.ThenChange(:init_qdu8_f32_qc8w_igemm_config_identifier) } static void init_qdu8_f32_qc4w_gemm_config(void) { + // LINT.IfChange(init_qdu8_f32_qc4w_gemm_config_identifier) + qdu8_f32_qc4w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qdu8_f32_qc4w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qdu8_f32_qc4w_gemm_config_config) + // LINT.IfChange(init_qdu8_f32_qc4w_gemm_config_config) // Use the same packing function throughout. qdu8_f32_qc4w_gemm_config.pack_weights_and_biases = (xnn_pack_weights_and_biases_fn) xnn_pack_qs4_weights_and_biases; qdu8_f32_qc4w_gemm_config.packed_stride_weights_and_biases = (xnn_packed_stride_weights_and_biases_fn) xnn_packed_stride_qs4_weights_and_biases; @@ -3427,9 +3551,14 @@ static void init_qdu8_f32_qc4w_gemm_config(void) { assert(qdu8_f32_qc4w_gemm_config.mr <= XNN_MAX_MR); assert(qdu8_f32_qc4w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); #endif //XNN_ARCH_X86 || XNN_ARCH_X86_64 + // LINT.ThenChange(:init_qdu8_f32_qc4w_gemm_config_identifier) } static void init_qd8_f32_qc8w_gemm_config(void) { + // LINT.IfChange(init_qd8_f32_qc8w_gemm_config_identifier) + qd8_f32_qc8w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qd8_f32_qc8w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qd8_f32_qc8w_gemm_config_config) + // LINT.IfChange(init_qd8_f32_qc8w_gemm_config_config) // Use the same packing function throughout. qd8_f32_qc8w_gemm_config.pack_weights_and_biases = (xnn_pack_weights_and_biases_fn)xnn_pack_qs8_weights_and_biases; @@ -3948,9 +4077,14 @@ static void init_qd8_f32_qc8w_gemm_config(void) { #endif assert(qd8_f32_qc8w_gemm_config.mr <= XNN_MAX_MR); assert(qd8_f32_qc8w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); + // LINT.ThenChange(:init_qd8_f32_qc8w_gemm_config_identifier) } static void init_qs8_qc4w_gemm_config(void) { + // LINT.IfChange(init_qs8_qc4w_gemm_config_identifier) + qs8_qc4w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qs8_qc4w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qs8_qc4w_gemm_config_config) + // LINT.IfChange(init_qs8_qc4w_gemm_config_config) #if XNN_ARCH_ARM64 && !XNN_PLATFORM_WINDOWS && XNN_ENABLE_ASSEMBLY const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); assert(hardware_config != NULL); @@ -4082,9 +4216,14 @@ static void init_qs8_qc4w_gemm_config(void) { } assert(qs8_qc4w_gemm_config.mr <= XNN_MAX_MR); assert(qs8_qc4w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1)); + // LINT.ThenChange(:init_qs8_qc4w_gemm_config_identifier) } static void init_qs8_qc8w_gemm_config(void) { + // LINT.IfChange(init_qs8_qc8w_gemm_config_identifier) + qs8_qc8w_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qs8_qc8w_gemm, /*version=*/0); + // LINT.ThenChange(:init_qs8_qc8w_gemm_config_config) + // LINT.IfChange(init_qs8_qc8w_gemm_config_config) // Use the same packing function throughout. qs8_qc8w_gemm_config.pack_weights_and_biases = (xnn_pack_weights_and_biases_fn)xnn_pack_qs8_weights_and_biases; @@ -4971,9 +5110,14 @@ static void init_qs8_qc8w_gemm_config(void) { qs8_qc8w_gemm_config.nr = 4; #endif assert(qs8_qc8w_gemm_config.mr <= XNN_MAX_MR); + // LINT.ThenChange(:init_qs8_qc8w_gemm_config_identifier) } static void init_qu8_gemm_config(void) { + // LINT.IfChange(init_qu8_gemm_config_identifier) + qu8_gemm_config.identifier = xnn_create_config_identifier(xnn_config_name_qu8_gemm, /*version=*/0); + // LINT.ThenChange(:init_qu8_gemm_config_config) + // LINT.IfChange(init_qu8_gemm_config_config) // Use the same packing function throughout. qu8_gemm_config.pack_weights_and_biases = (xnn_pack_weights_and_biases_fn)xnn_pack_qu8_weights_and_biases; @@ -5255,6 +5399,7 @@ static void init_qu8_gemm_config(void) { qu8_gemm_config.nr = 4; #endif assert(qu8_gemm_config.mr <= XNN_MAX_MR); + // LINT.ThenChange(:init_qu8_gemm_config_identifier) } const struct xnn_gemm_config* xnn_init_f16_gemm_config() { diff --git a/src/operators/fully-connected-nc.c b/src/operators/fully-connected-nc.c index 49adc81c80f..d0bc80dda9b 100644 --- a/src/operators/fully-connected-nc.c +++ b/src/operators/fully-connected-nc.c @@ -202,6 +202,7 @@ static enum xnn_status create_fully_connected_nc( cache_key.seed = cache_seed; cache_key.kernel = kernel; cache_key.bias = bias; + cache_key.config = &(gemm_config->identifier); if (use_weights_cache(fully_connected_op)) { cache_offset = xnn_weights_cache_look_up(fully_connected_op->weights_cache, &cache_key); diff --git a/src/xnnpack/config-types.h b/src/xnnpack/config-types.h index 7192a048585..c58b94bd376 100644 --- a/src/xnnpack/config-types.h +++ b/src/xnnpack/config-types.h @@ -10,6 +10,7 @@ #include #include +#include "include/experimental.h" #include "src/xnnpack/hardware-config.h" #include "src/xnnpack/microfnptr.h" @@ -162,6 +163,7 @@ struct xnn_ibilinear_chw_config { }; struct xnn_gemm_config { + xnn_config_identifier identifier; struct gemm_fused_ukernels minmax; struct gemm_fused_ukernels relu; struct gemm_fused_ukernels linear; diff --git a/src/xnnpack/config.h b/src/xnnpack/config.h index 6584c1da78c..d5968e56c1b 100644 --- a/src/xnnpack/config.h +++ b/src/xnnpack/config.h @@ -10,6 +10,7 @@ #include #include +#include "include/experimental.h" #include "src/xnnpack/common.h" #include "src/xnnpack/config-types.h" #include "src/xnnpack/microfnptr.h" @@ -359,17 +360,14 @@ static inline bool xnn_is_hmp_igemm_ukernel( XNN_INTERNAL const struct xnn_gemm_config* xnn_init_bf16_f32_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_f16_gemm_config(); -XNN_INTERNAL const struct xnn_gemm_config* xnn_init_f32_gemm_config( - uint32_t flags); -XNN_INTERNAL const struct xnn_gemm_config* xnn_init_f32_gemm_nr2_config( - uint32_t flags); +XNN_INTERNAL const struct xnn_gemm_config* xnn_init_f32_gemm_config(uint32_t flags); +XNN_INTERNAL const struct xnn_gemm_config* xnn_init_f32_gemm_nr2_config(uint32_t flags); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_f32_igemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_f32_qc8w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_f32_qc4w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_pf16_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_pf32_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_pqs8_qc8w_gemm_config(); -XNN_INTERNAL const struct xnn_gemm_config* xnn_init_pqs8_qc8w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qd8_f16_qb4w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qd8_f16_qc4w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qd8_f16_qc8w_gemm_config(); @@ -385,8 +383,7 @@ XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qdu8_f16_qc8w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qdu8_f32_qc8w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qdu8_f32_qb4w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qdu8_f16_qc4w_gemm_config(); -XNN_INTERNAL const struct xnn_gemm_config* -xnn_init_qdu8_f32_qc8w_igemm_config(); +XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qdu8_f32_qc8w_igemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qs8_qc4w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qs8_qc8w_gemm_config(); XNN_INTERNAL const struct xnn_gemm_config* xnn_init_qu8_gemm_config(); @@ -431,6 +428,54 @@ XNN_INTERNAL const struct xnn_lut32norm_config* xnn_init_u8_lut32norm_config(); XNN_INTERNAL const struct xnn_unpool_config* xnn_init_x32_unpool_config(); +// Used to create configuration identifiers for microkernel configuration +// fingerprinting. +// +// Even though these values should not be used outside of XNNPack, the +// fingerprints can be accessed and serialized. This means that the values +// should not be changed as they will be compared to later XNNPack versions. +// +// They may be deprecated and new values can be added. +typedef enum xnn_config_name { + xnn_config_name_unknown = 0, + xnn_config_name_bf16_f32_gemm = 1, + xnn_config_name_f16_gemm = 2, + xnn_config_name_f32_gemm = 3, + xnn_config_name_f32_gemm_nr2 = 4, + xnn_config_name_f32_igemm = 5, + xnn_config_name_f32_qc8w_gemm = 6, + xnn_config_name_f32_qc4w_gemm = 7, + xnn_config_name_pf16_gemm = 8, + xnn_config_name_pf32_gemm = 9, + xnn_config_name_pqs8_qc8w_gemm = 10, + xnn_config_name_qd8_f16_qb4w_gemm = 11, + xnn_config_name_qd8_f16_qc4w_gemm = 12, + xnn_config_name_qd8_f16_qc8w_gemm = 13, + xnn_config_name_qd8_f16_qc8w_igemm = 14, + xnn_config_name_qd8_f32_qb4w_gemm = 15, + xnn_config_name_qd8_f32_qc4w_gemm = 16, + xnn_config_name_qd8_f32_qc8w_gemm = 17, + xnn_config_name_qp8_f32_qc4w_gemm = 18, + xnn_config_name_qp8_f32_qc8w_gemm = 19, + xnn_config_name_qp8_f32_qb4w_gemm = 20, + xnn_config_name_qdu8_f32_qc4w_gemm = 21, + xnn_config_name_qdu8_f16_qc8w_gemm = 22, + xnn_config_name_qdu8_f32_qc8w_gemm = 23, + xnn_config_name_qdu8_f32_qb4w_gemm = 24, + xnn_config_name_qdu8_f16_qc4w_gemm = 25, + xnn_config_name_qdu8_f32_qc8w_igemm = 26, + xnn_config_name_qs8_qc4w_gemm = 27, + xnn_config_name_qs8_qc8w_gemm = 28, + xnn_config_name_qu8_gemm = 29, +} xnn_config_name; + +struct xnn_config_identifier xnn_create_config_identifier(xnn_config_name name, + uint32_t version); + +xnn_config_name xnn_get_config_name(const struct xnn_config_identifier* identifier); + +xnn_config_name xnn_get_config_version(const struct xnn_config_identifier* identifier); + #ifdef __cplusplus } // extern "C" #endif diff --git a/test/BUILD.bazel b/test/BUILD.bazel index 54b0f9d6508..560c01af19f 100644 --- a/test/BUILD.bazel +++ b/test/BUILD.bazel @@ -1020,6 +1020,15 @@ xnnpack_unit_test( ], ) +xnnpack_unit_test( + name = "config_test", + srcs = ["config.cc"], + deps = [ + "//:xnnpack_h", + "//src/configs:microkernel_configs", + ], +) + xnnpack_unit_test( name = "weights_cache_test", srcs = ["weights-cache.cc"], diff --git a/test/config.cc b/test/config.cc new file mode 100644 index 00000000000..b3540870f74 --- /dev/null +++ b/test/config.cc @@ -0,0 +1,22 @@ +#include "src/xnnpack/config.h" + +#include + +#include +#include +#include "include/experimental.h" + +namespace { + +using ::testing::Eq; + +TEST(ConfigIdentifierTest, CreateAndExtractConfigIdentifier) { + const int32_t expected_version = 3; + const xnn_config_name expected_name = xnn_config_name_f32_gemm; + const xnn_config_identifier identifier = + xnn_create_config_identifier(expected_name, expected_version); + EXPECT_THAT(xnn_get_config_name(&identifier), Eq(expected_name)); + EXPECT_THAT(xnn_get_config_version(&identifier), Eq(expected_version)); +} + +} // namespace