Add slice bm 04 (#3125)

* add slice bm

* fix slice bm

* fix slice run

* add slice insert

* fix slice run

* add slice bm case threshold

* add slice bm case threshold

Author: Zeref996

framework/slice_benchmark/run.py

@@ -3,17 +3,18 @@
 # @author Zeref996
 # encoding=utf-8 vi:ts=4:sw=4:expandtab:ft=python
 """
-slice 测试启动 
+slice 测试启动
 """
 
 import os
 import json
 import shutil
-import yaml
 from datetime import datetime
+import yaml
 from time_count_engine import SliceBenchMark
 from db.mysql_helper import SliceBenchmarkDB
 from db.snapshot import Snapshot
+from slicebm_utils.threshold import perf_compare
 
 
 class SliceTestRun(object):
@@ -102,7 +103,7 @@ def run_test_and_insert_data(self, comment, base):
             "update_time": test_complete_time,
         }
         db.update(table="slice_job", data=update_data, condition=f"id = {task_id}")
-        return res_dict, fail_cases_list
+        return res_dict, fail_cases_list, task_id
 
     def get_baseline(self, framework="paddle"):
         """
@@ -130,7 +131,7 @@ def ci_test(self):
         """
         ci 测试
         """
-        latest_res_dict, fail_cases_list = self.run_test_and_insert_data(comment="slice测试CI任务", base=0)
+        latest_res_dict, fail_cases_list, task_id = self.run_test_and_insert_data(comment="slice测试CI任务", base=0)
         baseline_res_dict = self.get_baseline(framework="paddle")
         print("开始使用本次CI测试结果, 与paddle基线进行性能对比 =============================>")
         perf_compare_res_dict, fail_perf_dict = self.res_dict_compare(baseline_res_dict, latest_res_dict)
@@ -150,50 +151,23 @@ def ci_test(self):
             print(f"slice测试失败, 存在功能失败case, 失败case有: {fail_cases_list}")
         if len(fail_perf_dict) > 0:
             print(f"slice测试失败, 存在性能下降case, 失败case性能变化: {fail_perf_dict}")
+
+        with open(self.db_config, encoding="utf-8") as f:
+            db_config = yaml.load(f, Loader=yaml.FullLoader)
+        db = SliceBenchmarkDB(**db_config["Config"]["slice_benchmark"]["MYSQL"])
         if len(fail_cases_list) + len(fail_perf_dict) > 0:
+            update_data = {
+                "result": "失败",
+                "update_time": self.timestamp(),
+            }
+            db.update(table="slice_job", data=update_data, condition=f"id = {task_id}")
             raise Exception("slice测试失败")
-
-    def perf_grade(self, res):
-        """
-        评分标准
-        :param res: 性能对比结果
-        :return:
-        """
-        grade = ""
-        if isinstance(res, str):
-            grade = res
-        else:
-            if res <= -0.2:
-                grade = "worse"
-            elif -0.2 < res <= -0.1:
-                grade = "doubt"
-            elif -0.1 < res <= 0.1:
-                grade = "equal"
-            elif res > 0.1:
-                grade = "better"
-        return grade
-
-    def perf_compare(self, baseline, latest):
-        """
-        比较函数
-        :param latest: 待测值
-        :param baseline: 基线值
-        :return: 比例值
-        """
-        if isinstance(baseline, str) or isinstance(baseline, str):
-            res = "error"
-            return res
         else:
-            if baseline == 0 or latest == 0:
-                res = 0
-            else:
-                if latest > baseline:
-                    res = (latest - baseline) / baseline * -1
-                else:
-                    res = (baseline - latest) / latest
-        grade = self.perf_grade(res)
-        return res, grade
-        # return "{:.2f}%".format(res * 100)
+            update_data = {
+                "result": "成功",
+                "update_time": self.timestamp(),
+            }
+            db.update(table="slice_job", data=update_data, condition=f"id = {task_id}")
 
     def res_dict_compare(self, baseline_res_dict, latest_res_dict):
         """
@@ -203,7 +177,7 @@ def res_dict_compare(self, baseline_res_dict, latest_res_dict):
         perf_compare_res_dict = {}
         for case_name, perf_value in latest_res_dict.items():
             if case_name in baseline_res_dict:
-                perf_compare_res, grade = self.perf_compare(baseline_res_dict[case_name], perf_value)
+                perf_compare_res, grade = perf_compare(baseline_res_dict[case_name], perf_value, case_name)
                 if grade == "worse" or grade == "doubt":
                     fail_perf_dict[case_name] = perf_compare_res
                 perf_compare_res_dict[case_name] = perf_compare_res
@@ -215,7 +189,7 @@ def res_dict_compare(self, baseline_res_dict, latest_res_dict):
                 print(f"{case_name}: 基线数据不存在, 本次测试数据{perf_value}, 无对比值")
 
         return perf_compare_res_dict, fail_perf_dict
-    
+
     def torch_res_dict_compare(self, torch_res_dict, latest_res_dict):
         """
         性能字典数据对比
@@ -225,7 +199,7 @@ def torch_res_dict_compare(self, torch_res_dict, latest_res_dict):
         for case_name_origin, perf_value in latest_res_dict.items():
             case_name = case_name_origin.replace("paddle", "torch")
             if case_name in torch_res_dict:
-                perf_compare_res, grade = self.perf_compare(torch_res_dict[case_name], perf_value)
+                perf_compare_res, grade = perf_compare(torch_res_dict[case_name], perf_value, case_name)
                 if grade == "worse" or grade == "doubt":
                     fail_perf_dict[case_name] = perf_compare_res
                 perf_compare_res_dict[case_name] = perf_compare_res

framework/slice_benchmark/slicebm_utils/threshold.py

@@ -0,0 +1,60 @@
+#!/bin/env python3
+# -*- coding: utf-8 -*-
+# @author Zeref996
+# encoding=utf-8 vi:ts=4:sw=4:expandtab:ft=python
+"""
+case性能阈值
+"""
+
+threshold_map = {
+    'Getitem - forward - Scalar - Integer - float16 - paddle' : [-0.25, -0.15, 0.15],
+    'Getitem - forward - Scalar - Tuple of Integers - float16 - paddle': [-0.25, -0.15, 0.15],
+    'basic_threshold': [-0.2, -0.1, 0.1],
+}
+
+def perf_grade(res, threshold):
+    """
+    评分标准
+    :param res: 性能对比结果
+    :return:
+    """
+    grade = ""
+    if isinstance(res, str):
+        grade = res
+    else:
+        if res <= threshold[0]:
+            grade = "worse"
+        elif threshold[0] < res <= threshold[1]:
+            grade = "doubt"
+        elif threshold[1] < res <= threshold[2]:
+            grade = "equal"
+        elif res > threshold[2]:
+            grade = "better"
+    return grade
+
+
+def perf_compare(baseline, latest, case_name):
+    """
+    比较函数
+    :param latest: 待测值
+    :param baseline: 基线值
+    :return: 比例值
+    """
+    if case_name in threshold_map:
+        threshold = threshold_map[case_name]
+    else:
+        threshold = threshold_map['basic_threshold']
+
+    if isinstance(baseline, str) or isinstance(baseline, str):
+        res = "error"
+        return res
+    else:
+        if baseline == 0 or latest == 0:
+            res = 0
+        else:
+            if latest > baseline:
+                res = (latest - baseline) / baseline * -1
+            else:
+                res = (baseline - latest) / latest
+    grade = perf_grade(res, threshold)
+    return res, grade