LLM pipeline implementation

.bazelrc

@@ -10,6 +10,10 @@ build --spawn_strategy=standalone
 # This flag is required by tensorflow
 common --experimental_repo_remote_exec
 
+# Without these, tensorflow complains about lack of CUDA library.
+common --repo_env=TF_NEED_CUDA=0
+common --repo_env=TF_NEED_ROCM=0
+
 # Default options should come above this line.
 
 # Configure logs
@@ -18,13 +22,16 @@ build:verbose_logs --output_filter=
 
 # Suppress C++ compiler warnings, otherwise build logs become 10s of MBs.
 build:android --copt=-w
+build:linux --copt=-w
 build:ios --copt=-w
 build:windows --copt=/W0
 
 # Build in C++ 17 mode.
 build --cxxopt=-std=c++17
 build:android --cxxopt=-std=c++17
 build:android --host_cxxopt=-std=c++17
+build:linux --cxxopt=-std=c++17
+build:linux --host_cxxopt=-std=c++17
 build:ios --cxxopt=-std=c++17
 build:ios --host_cxxopt=-std=c++17
 build:ios --cxxopt=-xobjective-c++
@@ -41,6 +48,33 @@ build:android_x86_64 --config=android
 build:android_x86_64 --cpu=x86_64
 build:android_x86_64 --fat_apk_cpu=x86_64
 
+
+build:android_x86_64 --define=xnn_enable_avx512fp16=false
+build:android_x86_64 --define=xnn_enable_avxvnniint8=false
+
+# Linux configs
+build:linux_x86_64 --config=linux
+build:linux_x86_64 --cpu=k8
+# Not required, but enables the proper SSE/MMX instructions per CPU
+build:linux_x86_64 --copt=-march=native
+
+# These may be neccessary depending on CPU instruction support
+#build:linux_x86_64 --define=xnn_enable_avx=false
+#build:linux_x86_64 --define=xnn_enable_avx2=false
+#build:linux_x86_64 --define=xnn_enable_avx512=false
+build:linux_x86_64 --define=xnn_enable_avx512fp16=false
+#build:linux_x86_64 --define=xnn_enable_avxvnni=false
+build:linux_x86_64 --define=xnn_enable_avxvnniint8=false
+#build:linux_x86_64 --define=xnn_enable_vnni=false
+
+
+# Optional, enable for debugging or compilation errors
+#build:linux_x86_64 --action_env=CC=gcc
+#build:linux_x86_64 --action_env=CXX=g++
+#build:linux_x86_64 --strip=never
+#build:linux_x86_64 --copt=-fno-omit-frame-pointer
+#build:linux_x86_64 --linkopt=-fno-omit-frame-pointer
+
 # iOS configs
 build:ios --apple_platform_type=ios
 build:ios --copt=-Wno-c++11-narrowing

WORKSPACE

@@ -38,14 +38,44 @@ http_archive(
         "//:flutter/third_party/use_unsigned_char.patch",
         # Fix tensorflow not being able to read image files on Windows
         "//:flutter/third_party/tensorflow-fix-file-opening-mode-for-Windows.patch",
-        "//:flutter/third_party/tf-eigen.patch",
-        # NDK 25 support
-        "//patches:ndk_25_r14.diff",
+        #"//:flutter/third_party/tf-eigen.patch",
     ] + PATCH_FILE,
-    sha256 = "ce357fd0728f0d1b0831d1653f475591662ec5bca736a94ff789e6b1944df19f",
-    strip_prefix = "tensorflow-2.14.0",
+    sha256 = "d7876f4bb0235cac60eb6316392a7c48676729860da1ab659fb440379ad5186d",
+    strip_prefix = "tensorflow-2.18.0",
     urls = [
-        "https://github.com/tensorflow/tensorflow/archive/v2.14.0.tar.gz",
+        "https://github.com/tensorflow/tensorflow/archive/v2.18.0.tar.gz",
+    ],
+)
+
+load("@org_tensorflow//third_party/gpus:cuda_configure.bzl", "cuda_configure")
+
+cuda_configure(name = "local_config_cuda")
+
+load("@org_tensorflow//third_party/gpus:rocm_configure.bzl", "rocm_configure")
+
+rocm_configure(name = "local_config_rocm")
+
+http_archive(
+    name = "com_google_sentencepiece",
+    build_file = "@//patches:sentencepiece.BUILD",
+    patch_args = ["-p1"],
+    patches = ["@//patches:com_google_sentencepiece.diff"],
+    sha256 = "8409b0126ebd62b256c685d5757150cf7fcb2b92a2f2b98efb3f38fc36719754",
+    strip_prefix = "sentencepiece-0.1.96",
+    urls = [
+        "https://github.com/google/sentencepiece/archive/refs/tags/v0.1.96.zip",
+    ],
+)
+
+http_archive(
+    name = "darts_clone",
+    build_file = "@//patches:darts_clone.BUILD",
+    patch_args = ["-p0"],
+    patches = ["//patches:darts_no_exceptions.diff"],
+    sha256 = "c97f55d05c98da6fcaf7f9ecc6a6dc6bc5b18b8564465f77abff8879d446491c",
+    strip_prefix = "darts-clone-e40ce4627526985a7767444b6ed6893ab6ff8983",
+    urls = [
+        "https://github.com/s-yata/darts-clone/archive/e40ce4627526985a7767444b6ed6893ab6ff8983.zip",
     ],
 )
 

flutter/android/android-docker.mk

@@ -19,7 +19,7 @@ user_id=$(shell id -u)
 .PHONY: flutter/android/docker/image
 flutter/android/docker/image: output/docker/mlperf_mobile_flutter_android_${user_id}.stamp
 output/docker/mlperf_mobile_flutter_android_${user_id}.stamp: flutter/android/docker/Dockerfile
-	docker image build -t ${DOCKER_IMAGE_TAG} flutter/android/docker
+	DOCKER_BUILDKIT=1 docker buildx build --tag ${DOCKER_IMAGE_TAG} flutter/android/docker
 	mkdir -p output/docker
 	touch $@
 
@@ -68,4 +68,4 @@ docker/flutter/android/release: flutter/check-release-env flutter/android/docker
 docker/flutter/clean: flutter/check-release-env
 	MSYS2_ARG_CONV_EXCL="*" docker run \
 		${flutter_common_docker_flags} \
-		make flutter/clean
+		make flutter/clean

flutter/assets/tasks.pbtxt

@@ -336,6 +336,59 @@ task {
   }
 }
 
+task {
+  id: "llm"
+  name: "LLM"
+  max_throughput: 2000
+  max_accuracy: 1.0
+  scenario: "SingleStream"
+  runs {
+    normal {
+      min_query_count: 100
+      min_duration: 60
+      max_duration: 300
+    }
+    quick {
+      min_query_count: 10
+      min_duration: 10
+      max_duration: 40
+    }
+    rapid {
+      min_query_count: 6
+      min_duration: 6
+      max_duration: 60
+    }
+  }
+  datasets {
+    type: MMLU
+      full {
+      name: "TinyMMLU prompt set for LLM"
+      input_path: "local:///mlperf_datasets/tinymmlu/data.tfrecord"
+      input_checksum: "c20f9115582217af15e4d9955b41ace1"
+      groundtruth_path: ""
+      groundtruth_checksum: ""
+    }
+    lite {
+      name: "TinyMMLU prompt set for LLM"
+      input_path: "local:///mlperf_datasets/tinymmlu/data.tfrecord"
+      input_checksum: "c20f9115582217af15e4d9955b41ace1"
+      groundtruth_path: ""
+      groundtruth_checksum: ""
+    }
+    tiny {
+      name: "TinyMMLU prompt set for LLM"
+      input_path: "local:///mlperf_datasets/tinymmlu/data.tfrecord"
+      input_checksum: "c20f9115582217af15e4d9955b41ace1"
+      groundtruth_path: ""
+      groundtruth_checksum: ""
+    }
+  }
+  model {
+    id: "LLM"
+    name: "LLM"
+  }
+}
+
 task {
   id: "stable_diffusion"
   name: "Stable Diffusion"

flutter/cpp/backend.h

@@ -45,7 +45,8 @@ class Backend {
   virtual const std::string& AcceleratorName() const = 0;
 
   // Run inference for a sample. Inputs is already set by SetInputs.
-  virtual void IssueQuery() = 0;
+  // TODO might be good to provide the callback and context along with the inputs if possible
+  virtual void IssueQuery(ft_callback callback, void* context) = 0;
 
   // Flush the staged queries immediately.
   virtual void FlushQueries() = 0;

flutter/cpp/backends/external.h

@@ -48,7 +48,7 @@ struct BackendFunctions {
       std::add_pointer<const char*(mlperf_backend_ptr_t)>::type;
   using BackendDeletePtr = std::add_pointer<void(mlperf_backend_ptr_t)>::type;
   using IssueQueryPtr =
-      std::add_pointer<mlperf_status_t(mlperf_backend_ptr_t)>::type;
+      std::add_pointer<mlperf_status_t(mlperf_backend_ptr_t, ft_callback, void*)>::type;
   using FlushQueriesPtr =
       std::add_pointer<mlperf_status_t(mlperf_backend_ptr_t)>::type;
 
@@ -157,8 +157,8 @@ class ExternalBackend : public Backend {
   }
 
   // Run inference for a sample.
-  void IssueQuery() override {
-    if (backend_functions_.issue_query(backend_ptr_) != MLPERF_SUCCESS) {
+  void IssueQuery(ft_callback callback, void* context) override {
+    if (backend_functions_.issue_query(backend_ptr_, callback, context) != MLPERF_SUCCESS) {
       LOG(FATAL) << "Error while inferencing model";
     }
   }

flutter/cpp/binary/BUILD

@@ -55,6 +55,7 @@ cc_binary(
         "//flutter/cpp/datasets:coco",
         "//flutter/cpp/datasets:coco_gen",
         "//flutter/cpp/datasets:imagenet",
+        "//flutter/cpp/datasets:mmlu_gen",
         "//flutter/cpp/datasets:snu_sr",
         "//flutter/cpp/datasets:squad",
         "//flutter/cpp/proto:mlperf_task_cc_proto",

flutter/cpp/binary/cmdline-docker.mk

@@ -17,4 +17,10 @@
 docker/cmdline/android/release: flutter/android/docker/image
 	MSYS2_ARG_CONV_EXCL="*" docker run \
 		${flutter_common_docker_flags} \
-		make cmdline/android/bins/release
+		make cmdline/android/bins/release
+
+.PHONY: docker/cmdline/linux/release
+docker/cmdline/linux/release: flutter/android/docker/image
+	MSYS2_ARG_CONV_EXCL="*" docker run \
+		${flutter_common_docker_flags} \
+		make cmdline/linux/bins/release

flutter/cpp/binary/cmdline.mk

@@ -16,6 +16,7 @@
 include flutter/cpp/binary/cmdline-docker.mk
 
 cmdline/android/bins/release: cmdline/android/libs/deps cmdline/android/bins/build cmdline/android/bins/copy
+cmdline/linux/bins/release: cmdline/linux/bins/build cmdline/linux/bins/copy
 
 .PHONY: cmdline/android/libs/deps
 cmdline/android/libs/deps:
@@ -52,6 +53,29 @@ cmdline/android/bins/copy:
 		@# macos doesn't support --recursive flag
 		chmod -R 777 ${cmdline_android_bin_release_path}
 
+.PHONY: cmdline/linux/bins/build
+cmdline/linux/bins/build:
+	bazel ${BAZEL_OUTPUT_ROOT_ARG} ${proxy_bazel_args} ${sonar_bazel_startup_options} \
+		build ${BAZEL_CACHE_ARG} ${bazel_links_arg} ${sonar_bazel_build_args} \
+		--config=linux_x86_64 \
+		${backend_tflite_android_target} \
+		//flutter/cpp/flutter:libbackendbridge.so \
+		//flutter/cpp/binary:main
+
+cmdline_linux_bin_release_path=output/linux/cmdline
+.PHONY: cmdline/linux/bins/copy
+cmdline/linux/bins/copy:
+	rm -rf ${cmdline_linux_bin_release_path}
+	mkdir -p ${cmdline_linux_bin_release_path}
+	@# macos doesn't support --target-directory flag
+	cp -f \
+		${backend_tflite_android_files} \
+		${BAZEL_LINKS_PREFIX}bin/flutter/cpp/flutter/libbackendbridge.so \
+		${BAZEL_LINKS_PREFIX}bin/flutter/cpp/binary/main \
+		${cmdline_linux_bin_release_path}
+		@# macos doesn't support --recursive flag
+		chmod -R 777 ${cmdline_linux_bin_release_path}
+
 windows_cmdline_folder=output/windows/cmdline
 .PHONY: cmdline/windows/bins
 cmdline/windows/bins:
@@ -92,4 +116,4 @@ cmdline/windows/prepare-dlls:
 .PHONY: cmdline/windows/copy-dlls
 cmdline/windows/copy-dlls:
 	currentDir=$$(pwd) && cd "${msvc_arm_dlls_path}" && \
-		cp  --target-directory $$currentDir/${windows_cmdline_folder} ${msvc_arm_dlls_list}		
+		cp  --target-directory $$currentDir/${windows_cmdline_folder} ${msvc_arm_dlls_list}