[GPU] Multi kernel gemm for TILE_K_NOT_DIVISIBLE and TILE_N_NOT_DIVISIBLE (#24007)

### Details:
- In the case of shape-agnostic gemm kernel, `TILE_K_NOT_DIVISIBLE` and
`TILE_N_NOT_DIVISIBLE` can be expressed as a conditional branches rather
than constant values, in which case performance could be degraded.
- This PR changes a gemm impl to have four OpenCL kernels with
`TILE_K_NOT_DIVISIBLE` and `TILE_N_NOT_DIVISIBLE` of 0 and 1. Just
before enqueuing a gemm kernel, it checks the value of
`TILE_K_NOT_DIVISIBLE` and `TILE_N_NOT_DIVISIBLE` on the host side, and
chooses one of them.

### Tickets:
 - 134699

Author: e-ddykim

src/plugins/intel_gpu/src/kernel_selector/cl_kernels/gemm_tiled_opt.cl

@@ -270,15 +270,11 @@ KERNEL(gemm_tiled_opt)(
                 b_tile[b_load_id] = b_raw_global_id > N - 1 ? 0 : b_ptr[sglid];
                 #else // B_VEC_SIZE == 1
                     #if TILE_N_NOT_DIVISIBLE
-                if (TILE_N_NOT_DIVISIBLE_CALC) {
-                    unroll_for (uint b_elem = 0; b_elem < B_VEC_SIZE; ++b_elem) {
-                        b_tile[b_load_id][b_elem] = b_ptr[sglid + SIMD_WIDTH * b_elem];
-                    }
-                } else {
-                    b_tile[b_load_id] = BLOCK_READ_B(b_ptr, 0);
-                }
+                        unroll_for (uint b_elem = 0; b_elem < B_VEC_SIZE; ++b_elem) {
+                            b_tile[b_load_id][b_elem] = b_ptr[sglid + SIMD_WIDTH * b_elem];
+                        }
                     #else // TILE_N_NOT_DIVISIBLE
-                b_tile[b_load_id] = BLOCK_READ_B(b_ptr, 0);
+                        b_tile[b_load_id] = BLOCK_READ_B(b_ptr, 0);
                     #endif // TILE_N_NOT_DIVISIBLE
                 #endif // B_VEC_SIZE == 1
                 b_ptr += input1_offset;
@@ -381,11 +377,7 @@ KERNEL(gemm_tiled_opt)(
 
         // Loading A tile and tile C calculation
 #if IS_DYNAMIC && !INDIRECT_INPUT0 && !HAS_DYNAMIC_K_PADDING && TRANSPOSE_INPUT0 == TRANSPOSE_X_LAST
-    #if TILE_K_NOT_DIVISIBLE
-            A_FLOATN a_read = TILE_K_NOT_DIVISIBLE_CALC ? a_ptr[sglid] : BLOCK_READ_A(a_ptr, 0);
-    #else
-            A_FLOATN a_read = BLOCK_READ_A(a_ptr, 0);
-    #endif
+        A_FLOATN a_read = TILE_K_NOT_DIVISIBLE ? a_ptr[sglid] : BLOCK_READ_A(a_ptr, 0);
 #endif
         unroll_for (uint dot_id = 0; dot_id < tile_m_iterations; dot_id++) {
 #if TRANSPOSE_INPUT0 == TRANSPOSE_X_LAST
@@ -431,11 +423,7 @@ KERNEL(gemm_tiled_opt)(
             }
     #if IS_DYNAMIC && !INDIRECT_INPUT0 && !HAS_DYNAMIC_K_PADDING
         // Read A for next dot_id
-        #if TILE_K_NOT_DIVISIBLE
-            a_read = (dot_id + 1 < tile_m_iterations) ? TILE_K_NOT_DIVISIBLE_CALC ? a_ptr[sglid] : BLOCK_READ_A(a_ptr, 0) : 0;
-        #else
-            a_read = (dot_id + 1 < tile_m_iterations) ? BLOCK_READ_A(a_ptr, 0) : 0;
-        #endif
+        a_read = (dot_id + 1 < tile_m_iterations) ? TILE_K_NOT_DIVISIBLE ? a_ptr[sglid] : BLOCK_READ_A(a_ptr, 0) : 0;
     #endif
 #elif TRANSPOSE_INPUT0 == TRANSPOSE_OTHER // TRANSPOSE_INPUT0
     #if INDIRECT_INPUT0
@@ -482,9 +470,8 @@ KERNEL(gemm_tiled_opt)(
     // Full tile calculation end
 
     // Handle leftovers for K
-#if TILE_K_NOT_DIVISIBLE
-    #if IS_DYNAMIC
-    if (TILE_K_NOT_DIVISIBLE_CALC) {
+#if IS_DYNAMIC
+    if (TILE_K_NOT_DIVISIBLE) {
         // Loading leftovers of the matrix B
         #if TRANSPOSE_INPUT1 != TRANSPOSE_Y_LAST
         B_FLOATN b_tile[TILE_K];
@@ -520,15 +507,11 @@ KERNEL(gemm_tiled_opt)(
                     b_tile[b_load_id] = b_raw_global_id > N - 1 ? 0 : b_ptr[sglid];
                 #else // B_VEC_SIZE == 1
                     #if TILE_N_NOT_DIVISIBLE
-                    if (TILE_N_NOT_DIVISIBLE_CALC) {
                         unroll_for (uint b_elem = 0; b_elem < B_VEC_SIZE; ++b_elem) {
                             b_tile[b_load_id][b_elem] = b_ptr[sglid + SIMD_WIDTH * b_elem];
                         }
-                    } else {
-                        b_tile[b_load_id] = BLOCK_READ_B(b_ptr, 0);
-                    }
                     #else
-                    b_tile[b_load_id] = BLOCK_READ_B(b_ptr, 0);
+                        b_tile[b_load_id] = BLOCK_READ_B(b_ptr, 0);
                     #endif // TILE_N_NOT_DIVISIBLE
                 #endif // B_VEC_SIZE == 1
                     b_ptr += input1_offset;
@@ -698,8 +681,7 @@ KERNEL(gemm_tiled_opt)(
             c_tile[dot_id] = mad((INPUT0_TYPE)(sub_group_broadcast(a_read, simd_id)), b_tile[simd_id], c_tile[dot_id]);
         }
     } // Loading leftovers of the matrix A and tile C calculation end
-    #endif // IS_DYNAMIC
-#endif // TILE_K_NOT_DIVISIBLE
+#endif // IS_DYNAMIC
 
 #if HAS_FUSED_OPS && FUSED_OPS_CAN_USE_PRELOAD
     #if IS_DYNAMIC

src/plugins/intel_gpu/src/kernel_selector/kernels/gemm/gemm_kernel_tiled_opt.cpp

@@ -156,20 +156,6 @@ JitConstants GemmKernelTiledOpt::GetJitConstants(const gemm_params& params) cons
         const std::string not_divisible_k = "(" + leftover_k + "!=0)";
         const std::string full_iteration_k = "(" + k_size + "/" + std::to_string(tuning_data.tile_k_size) + ")";
 
-        bool tile_k_may_have_leftover = false;
-        if (k_size.find("shape_info") == std::string::npos) {
-            tile_k_may_have_leftover = ((std::stoi(k_size) % tuning_data.tile_k_size) != 0);
-        } else {
-            tile_k_may_have_leftover = true;
-        }
-
-        bool tile_n_may_have_leftover = false;
-        if (n_size.find("shape_info") == std::string::npos) {
-            tile_n_may_have_leftover = ((std::stoi(n_size) % tuning_data.tile_n_size) != 0);
-        } else {
-            tile_n_may_have_leftover = true;
-        }
-
         jit.AddConstants({
             MakeJitConstant("M", m_size),
             MakeJitConstant("K", k_size),
@@ -182,10 +168,8 @@ JitConstants GemmKernelTiledOpt::GetJitConstants(const gemm_params& params) cons
             MakeJitConstant("TILE_N", tuning_data.tile_n_size),
             MakeJitConstant("K_FULL_ITERATIONS", full_iteration_k),
             MakeJitConstant("TILE_M_NOT_DIVISIBLE", not_divisible_m),
-            MakeJitConstant("TILE_K_NOT_DIVISIBLE", tile_k_may_have_leftover),
-            MakeJitConstant("TILE_K_NOT_DIVISIBLE_CALC", not_divisible_k),
-            MakeJitConstant("TILE_N_NOT_DIVISIBLE", tile_n_may_have_leftover),
-            MakeJitConstant("TILE_N_NOT_DIVISIBLE_CALC", not_divisible_n),
+            MakeJitConstant("TILE_K_NOT_DIVISIBLE", not_divisible_k),
+            MakeJitConstant("TILE_N_NOT_DIVISIBLE", not_divisible_n),
             MakeJitConstant("TILE_M_LEFTOVER", leftover_m),
             MakeJitConstant("TILE_K_LEFTOVER", leftover_k),
             MakeJitConstant("TILE_N_LEFTOVER", leftover_n),
@@ -356,7 +340,54 @@ JitConstants GemmKernelTiledOpt::GetJitConstants(const gemm_params& params) cons
 }
 
 KernelsData GemmKernelTiledOpt::GetKernelsData(const Params& params) const {
-    return GetCommonKernelsData(params);
+    if (!Validate(params)) {
+        return KernelsData();
+    }
+
+    const auto& prim_params = static_cast<const gemm_params&>(params);
+    size_t num_kernels = params.is_shape_agnostic ? 4 : 1;
+    auto dispatchData = SetDefault(prim_params);
+    KernelData k_data = KernelData::Default<gemm_params>(params, num_kernels);
+    GetUpdateDispatchDataFunc(k_data);
+    auto cldnn_jit = GetJitConstants(prim_params);
+    for (size_t i = 0; i < num_kernels; i++) {
+        if (params.is_shape_agnostic) {
+            cldnn_jit.RemoveConstant("TILE_K_NOT_DIVISIBLE");
+            cldnn_jit.RemoveConstant("TILE_N_NOT_DIVISIBLE");
+            if (i == 0) {
+                cldnn_jit.AddConstant(MakeJitConstant("TILE_K_NOT_DIVISIBLE", "0"));
+                cldnn_jit.AddConstant(MakeJitConstant("TILE_N_NOT_DIVISIBLE", "0"));
+            } else if (i == 1) {
+                cldnn_jit.AddConstant(MakeJitConstant("TILE_K_NOT_DIVISIBLE", "0"));
+                cldnn_jit.AddConstant(MakeJitConstant("TILE_N_NOT_DIVISIBLE", "1"));
+            } else if (i == 2) {
+                cldnn_jit.AddConstant(MakeJitConstant("TILE_K_NOT_DIVISIBLE", "1"));
+                cldnn_jit.AddConstant(MakeJitConstant("TILE_N_NOT_DIVISIBLE", "0"));
+            } else if (i == 3) {
+                cldnn_jit.AddConstant(MakeJitConstant("TILE_K_NOT_DIVISIBLE", "1"));
+                cldnn_jit.AddConstant(MakeJitConstant("TILE_N_NOT_DIVISIBLE", "1"));
+            }
+        }
+        auto entry_point = GetEntryPoint(kernelName, prim_params.layerID, params, i);
+        auto jit = CreateJit(kernelName, cldnn_jit, entry_point);
+
+        auto& kernel = k_data.kernels[i];
+        FillCLKernelData(kernel,
+                        dispatchData,
+                        params.engineInfo,
+                        kernelName,
+                        jit,
+                        entry_point,
+                        EXE_MODE_DEFAULT,
+                        false,
+                        false,
+                        (uint32_t)prim_params.inputs.size(),
+                        GetFusedPrimitiveInputsCount(params),
+                        1,
+                        prim_params.is_shape_agnostic);
+    }
+
+    return {k_data};
 }
 
 KernelsPriority GemmKernelTiledOpt::GetKernelsPriority(const Params& params) const {
@@ -404,4 +435,64 @@ bool GemmKernelTiledOpt::Validate(const Params& params) const {
 
     return true;
 }
+
+void GemmKernelTiledOpt::GetUpdateDispatchDataFunc(KernelData& kd) const {
+    if (kd.kernels.size() == 1) {
+        Parent::GetUpdateDispatchDataFunc(kd);
+    } else {
+        kd.update_dispatch_data_func = [this](const Params& params, KernelData& kd) {
+            const auto& prim_params = static_cast<const gemm_params&>(params);
+
+            auto getTensorValue = [](const DataTensor& t, const int64_t dim_idx) -> size_t {
+                switch (dim_idx) {
+                    case 1:
+                        return t.Feature().v;
+                    case 2:
+                        return t.U().v;
+                    case 3:
+                        return t.V().v;
+                    case 4:
+                        return t.W().v;
+                    case 5:
+                        return t.Z().v;
+                    case 6:
+                        return t.Y().v;
+                    case 7:
+                        return t.X().v;
+                    default:
+                        return t.Batch().v;
+                }
+            };
+
+            GemmTuningData tuning_data = SetTuningParams(prim_params);
+            auto input0_dims = ConvTo8dims(prim_params.input0_order);
+            auto input1_dims = ConvTo8dims(prim_params.input1_order);
+            auto k_size = getTensorValue(prim_params.inputs[0], input0_dims[7]);
+            auto n_size = getTensorValue(prim_params.inputs[1], input1_dims[7]);
+            bool not_divisible_k = ((k_size % tuning_data.tile_k_size) != 0);
+            bool not_divisible_n = ((n_size % tuning_data.tile_n_size) != 0);
+            size_t execute_kernel_idx = 0;
+            if (not_divisible_k == false && not_divisible_n == false) {
+                execute_kernel_idx = 0;
+            } else if (not_divisible_k == false && not_divisible_n == true) {
+                execute_kernel_idx = 1;
+            } else if (not_divisible_k == true && not_divisible_n == false) {
+                execute_kernel_idx = 2;
+            } else if (not_divisible_k == true && not_divisible_n == true) {
+                execute_kernel_idx = 3;
+            }
+
+            auto dispatchData = SetDefault(prim_params);
+            for (size_t i = 0; i < kd.kernels.size(); i++) {
+                kd.kernels[i].params.workGroups.global = dispatchData.gws;
+                kd.kernels[i].params.workGroups.local = dispatchData.lws;
+                if (execute_kernel_idx == i) {
+                    kd.kernels[i].skip_execution = KernelData::SkipKernelExecution(prim_params);
+                } else {
+                    kd.kernels[i].skip_execution = true;
+                }
+            }
+        };
+    }
+}
 }  // namespace kernel_selector

src/plugins/intel_gpu/src/kernel_selector/kernels/gemm/gemm_kernel_tiled_opt.h

@@ -36,5 +36,6 @@ class GemmKernelTiledOpt : public GemmKernelBase {
     JitConstants GetJitConstants(const gemm_params& params) const override;
     GemmTuningData SetTuningParams(const gemm_params& params) const;
     DeviceFeaturesKey get_required_device_features_key(const Params& params) const override;
+    void GetUpdateDispatchDataFunc(KernelData& kd) const override;
 };
 }  // namespace kernel_selector