From fe4679d0f2ced5b89b2f44026980fac11f904b38 Mon Sep 17 00:00:00 2001 From: "Zak, Pawel" Date: Mon, 8 Sep 2025 20:43:55 +0200 Subject: [PATCH 01/13] Introduce PipelineElementsSelector --- .../pipelines/smartnvr/pipeline.py | 285 ++++++++++++------ 1 file changed, 187 insertions(+), 98 deletions(-) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py index bc613956f..9be7dc2f3 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py @@ -7,6 +7,8 @@ from gstpipeline import GstPipeline from utils import UINT8_DTYPE_SIZE, VIDEO_STREAM_META_PATH, is_yolov10_model +logger = logging.getLogger("smartnvr") + class SmartNVRPipeline(GstPipeline): def __init__(self): @@ -20,7 +22,7 @@ def __init__(self): self._sink = "sink_{id}::xpos={xpos} sink_{id}::ypos={ypos} sink_{id}::alpha=1 " - # Add shmsink for live streaming (shared memory) + # Add shmsink for live-streaming (shared memory) self._shmsink = ( "shmsink socket-path=/tmp/shared_memory/video_stream " "wait-for-connection=false " @@ -137,14 +139,14 @@ def evaluate( if elements is None: elements = [] - # Set pre process backed for object detection + # Set pre-process backend for object detection parameters["object_detection_pre_process_backend"] = ( "opencv" if parameters["object_detection_device"] in ["CPU", "NPU"] else "va-surface-sharing" ) - # Set pre process backed for object classification + # Set pre-process backend for object classification parameters["object_classification_pre_process_backend"] = ( "opencv" if parameters["object_classification_device"] in ["CPU", "NPU"] @@ -162,100 +164,30 @@ def evaluate( ypos = 360 * (i // grid_size) sinks += self._sink.format(id=i, xpos=xpos, ypos=ypos) - # Find the available compositor in elements dynamically - if ( - parameters["object_detection_device"].startswith("GPU.") - and int(parameters["object_detection_device"].split(".")[1]) > 0 - ): - gpu_index = parameters["object_detection_device"].split(".")[1] - # Map GPU index to the corresponding VAAPI element suffix (e.g., "129" for GPU.1) - vaapi_suffix = str( - 128 + int(gpu_index) - ) # 128 + 1 = 129, 128 + 2 = 130, etc. - _compositor_element = f"varenderD{vaapi_suffix}compositor" - else: - _compositor_element = next( - ( - "vacompositor" - for element in elements - if element[1] == "vacompositor" - ), - next( - ( - "compositor" - for element in elements - if element[1] == "compositor" - ), - None, # Fallback to None if no compositor is found - ), - ) - - # Find the available encoder dynamically - if ( - parameters["object_detection_device"].startswith("GPU.") - and int(parameters["object_detection_device"].split(".")[1]) > 0 - ): - gpu_index = parameters["object_detection_device"].split(".")[1] - # Map GPU index to the corresponding VAAPI element suffix (e.g., "129" for GPU.1) - vaapi_suffix = str( - 128 + int(gpu_index) - ) # 128 + 1 = 129, 128 + 2 = 130, etc. - _encoder_element = f"varenderD{vaapi_suffix}h264lpenc" - else: - # Fallback to default encoder if no specific GPU is selected - _encoder_element = next( - ("vah264lpenc" for element in elements if element[1] == "vah264lpenc"), - next( - ("vah264enc" for element in elements if element[1] == "vah264enc"), - next( - ( - "x264enc bitrate=16000 speed-preset=superfast" - for element in elements - if element[1] == "x264enc" - ), - None, # Fallback to None if no encoder is found - ), - ), - ) - - # Find the available decoder and postprocessing elements dynamically - if ( - parameters["object_detection_device"].startswith("GPU.") - and int(parameters["object_detection_device"].split(".")[1]) > 0 + # Use PipelineElementsSelector for element selection + selector = PipelineElementsSelector(parameters, elements) + _compositor_element = selector.compositor_element() + _encoder_element = selector.encoder_element() + _decoder_element = selector.decoder_element() + _postprocessing_element = selector.postprocessing_element() + + # If any of the essential elements is not found, log an error and return an empty string + if not all( + [ + _compositor_element, + _encoder_element, + _decoder_element, + _postprocessing_element, + ] ): - # Extract the GPU index (e.g., "1" from "GPU.1") - gpu_index = parameters["object_detection_device"].split(".")[1] - # Map GPU index to the corresponding VAAPI element suffix (e.g., "129" for GPU.1) - vaapi_suffix = str( - 128 + int(gpu_index) - ) # 128 + 1 = 129, 128 + 2 = 130, etc. - _decoder_element = ( - f"varenderD{vaapi_suffix}h264dec ! video/x-raw(memory:VAMemory)" + logger.error("Could not find all necessary elements for the pipeline.") + logger.error( + f"Compositor: {_compositor_element}, Encoder: {_encoder_element}, Decoder: {_decoder_element}, Postprocessing: {_postprocessing_element}" ) - _postprocessing_element = f"varenderD{vaapi_suffix}postproc" + return "" else: - # Fallback to default elements if no specific GPU is selected - _decoder_element = next( - ( - "vah264dec ! video/x-raw(memory:VAMemory)" - for element in elements - if element[1] == "vah264dec" - ), - next( - ("decodebin" for element in elements if element[1] == "decodebin"), - None, # Fallback to None if no decoder is found - ), - ) - _postprocessing_element = next( - ("vapostproc" for element in elements if element[1] == "vapostproc"), - next( - ( - "videoscale" - for element in elements - if element[1] == "videoscale" - ), - None, # Fallback to None if no postprocessing is found - ), + logger.info( + f"Using pipeline elements - Compositor: {_compositor_element}, Encoder: {_encoder_element}, Decoder: {_decoder_element}, Postprocessing: {_postprocessing_element}" ) # Create the streams @@ -276,9 +208,9 @@ def evaluate( # Set inference config parameter for GPU if using YOLOv10 ie_config_parameter = "" - if parameters["object_detection_device"] == "GPU" and is_yolov10_model( - constants["OBJECT_DETECTION_MODEL_PATH"] - ): + if parameters.get("object_detection_device", "").startswith( + "GPU" + ) and is_yolov10_model(constants["OBJECT_DETECTION_MODEL_PATH"]): ie_config_parameter = "ie-config=GPU_DISABLE_WINOGRAD_CONVOLUTION=YES" streams += self._inference_stream_decode_detect_track.format( @@ -367,7 +299,7 @@ def evaluate( ) ) except Exception as e: - logging.warning(f"Could not write shared memory meta file: {e}") + logger.warning(f"Could not write shared memory meta file: {e}") streams = ( self._compositor_with_tee.format( @@ -395,3 +327,160 @@ def evaluate( # Evaluate the pipeline return "gst-launch-1.0 -q " + streams + + +class PipelineElementsSelector: + def __init__(self, parameters: dict, elements: list): + self.parameters = parameters + self.elements = elements + + # Calculate vaapi_suffix once + device = parameters.get("object_detection_device", "") + # If there is more than one GPU, device names are like GPU.0, GPU.1, ... + # If there is only one GPU, device name is just GPU + if device.startswith("GPU.") and int(device.split(".")[1]) > 0: + gpu_index = int(device.split(".")[1]) + self.vaapi_suffix = str( + 128 + gpu_index + ) # means that there is more than one GPU and GPU.N was selected (N>0), 128 + 1 = 129, 128 + 2 = 130, etc. + else: + self.vaapi_suffix = ( + None # means that either CPU, NPU, GPU, or GPU.0 was selected + ) + + # Compositor + # If vaapi_suffix is set, try to find varenderD{suffix}compositor first, e.g. varenderD129compositor for GPU.1 + # If not found, fallback to vacompositor or compositor + self._compositor_element = None + if self.vaapi_suffix is not None: + varender_compositor = f"varenderD{self.vaapi_suffix}compositor" + self._compositor_element = next( + ( + varender_compositor + for element in elements + if element[1] == varender_compositor + ), + None, + ) + if self._compositor_element is None: + self._compositor_element = next( + ( + "vacompositor" + for element in elements + if element[1] == "vacompositor" + ), + next( + ( + "compositor" + for element in elements + if element[1] == "compositor" + ), + None, + ), + ) + + # Encoder + # If vaapi_suffix is set, try to find varenderD{suffix}h264lpenc first, e.g. varenderD129h264lpenc for GPU.1 + # If not found, try varenderD{suffix}h264enc + # If still not found, fallback to vah264lpenc, vah264enc, or x264enc + self._encoder_element = None + if self.vaapi_suffix is not None: + varender_encoder_lp = f"varenderD{self.vaapi_suffix}h264lpenc" + varender_encoder = f"varenderD{self.vaapi_suffix}h264enc" + + self._encoder_element = next( + ( + varender_encoder_lp + for element in elements + if element[1] == varender_encoder_lp + ), + next( + ( + varender_encoder + for element in elements + if element[1] == varender_encoder + ), + None, + ), + ) + if self._encoder_element is None: + self._encoder_element = next( + ("vah264lpenc" for element in elements if element[1] == "vah264lpenc"), + next( + ("vah264enc" for element in elements if element[1] == "vah264enc"), + next( + ( + "x264enc bitrate=16000 speed-preset=superfast" + for element in elements + if element[1] == "x264enc" + ), + None, + ), + ), + ) + + # Decoder + # If vaapi_suffix is set, try to find varenderD{suffix}h264dec first, e.g. varenderD129h264dec for GPU.1 + # If not found, fallback to vah264dec or decodebin + self._decoder_element = None + if self.vaapi_suffix is not None: + varender_decoder = f"varenderD{self.vaapi_suffix}h264dec" + self._decoder_element = next( + ( + f"{varender_decoder} ! video/x-raw(memory:VAMemory)" + for element in elements + if element[1] == varender_decoder + ), + None, + ) + if self._decoder_element is None: + self._decoder_element = next( + ( + "vah264dec ! video/x-raw(memory:VAMemory)" + for element in elements + if element[1] == "vah264dec" + ), + next( + ("decodebin" for element in elements if element[1] == "decodebin"), + None, + ), + ) + + # Postprocessing + # If vaapi_suffix is set, try to find varenderD{suffix}postproc first, e.g. varenderD129postproc for GPU.1 + # If not found, fallback to vapostproc or videoscale + self._postprocessing_element = None + if self.vaapi_suffix is not None: + varender_postprocessing = f"varenderD{self.vaapi_suffix}postproc" + self._postprocessing_element = next( + ( + varender_postprocessing + for element in elements + if element[1] == varender_postprocessing + ), + None, + ) + if self._postprocessing_element is None: + self._postprocessing_element = next( + ("vapostproc" for element in elements if element[1] == "vapostproc"), + next( + ( + "videoscale" + for element in elements + if element[1] == "videoscale" + ), + None, + ), + ) + + def compositor_element(self): + return self._compositor_element + + def encoder_element(self): + return self._encoder_element + + def decoder_element(self): + return self._decoder_element + + def postprocessing_element(self): + return self._postprocessing_element From 55b6888202c31b6c4b1a529a9b12da1ab7eec26d Mon Sep 17 00:00:00 2001 From: "Zak, Pawel" Date: Mon, 8 Sep 2025 23:23:30 +0200 Subject: [PATCH 02/13] fix chosen device --- .../pipelines/smartnvr/pipeline.py | 152 +++++++++--------- 1 file changed, 74 insertions(+), 78 deletions(-) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py index 9be7dc2f3..2f03e4ee4 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py @@ -333,26 +333,35 @@ class PipelineElementsSelector: def __init__(self, parameters: dict, elements: list): self.parameters = parameters self.elements = elements + self.gpu_id = -1 + self.vaapi_suffix = None # Calculate vaapi_suffix once device = parameters.get("object_detection_device", "") - # If there is more than one GPU, device names are like GPU.0, GPU.1, ... + + # Determine gpu_id and vaapi_suffix # If there is only one GPU, device name is just GPU - if device.startswith("GPU.") and int(device.split(".")[1]) > 0: - gpu_index = int(device.split(".")[1]) - self.vaapi_suffix = str( - 128 + gpu_index - ) # means that there is more than one GPU and GPU.N was selected (N>0), 128 + 1 = 129, 128 + 2 = 130, etc. + # If there is more than one GPU, device names are like GPU.0, GPU.1, ... + if device == "GPU": + self.gpu_id = 0 + elif device.startswith("GPU."): + try: + gpu_index = int(device.split(".")[1]) + if gpu_index == 0: + self.gpu_id = 0 + elif gpu_index > 0: + self.vaapi_suffix = str(128 + gpu_index) + self.gpu_id = gpu_index + except (IndexError, ValueError): + self.gpu_id = -1 else: - self.vaapi_suffix = ( - None # means that either CPU, NPU, GPU, or GPU.0 was selected - ) + self.gpu_id = -1 - # Compositor - # If vaapi_suffix is set, try to find varenderD{suffix}compositor first, e.g. varenderD129compositor for GPU.1 - # If not found, fallback to vacompositor or compositor self._compositor_element = None - if self.vaapi_suffix is not None: + self._encoder_element = None + self._decoder_element = None + self._postprocessing_element = None + if self.gpu_id > 0: varender_compositor = f"varenderD{self.vaapi_suffix}compositor" self._compositor_element = next( ( @@ -362,32 +371,9 @@ def __init__(self, parameters: dict, elements: list): ), None, ) - if self._compositor_element is None: - self._compositor_element = next( - ( - "vacompositor" - for element in elements - if element[1] == "vacompositor" - ), - next( - ( - "compositor" - for element in elements - if element[1] == "compositor" - ), - None, - ), - ) - # Encoder - # If vaapi_suffix is set, try to find varenderD{suffix}h264lpenc first, e.g. varenderD129h264lpenc for GPU.1 - # If not found, try varenderD{suffix}h264enc - # If still not found, fallback to vah264lpenc, vah264enc, or x264enc - self._encoder_element = None - if self.vaapi_suffix is not None: varender_encoder_lp = f"varenderD{self.vaapi_suffix}h264lpenc" varender_encoder = f"varenderD{self.vaapi_suffix}h264enc" - self._encoder_element = next( ( varender_encoder_lp @@ -403,27 +389,7 @@ def __init__(self, parameters: dict, elements: list): None, ), ) - if self._encoder_element is None: - self._encoder_element = next( - ("vah264lpenc" for element in elements if element[1] == "vah264lpenc"), - next( - ("vah264enc" for element in elements if element[1] == "vah264enc"), - next( - ( - "x264enc bitrate=16000 speed-preset=superfast" - for element in elements - if element[1] == "x264enc" - ), - None, - ), - ), - ) - # Decoder - # If vaapi_suffix is set, try to find varenderD{suffix}h264dec first, e.g. varenderD129h264dec for GPU.1 - # If not found, fallback to vah264dec or decodebin - self._decoder_element = None - if self.vaapi_suffix is not None: varender_decoder = f"varenderD{self.vaapi_suffix}h264dec" self._decoder_element = next( ( @@ -433,44 +399,74 @@ def __init__(self, parameters: dict, elements: list): ), None, ) - if self._decoder_element is None: - self._decoder_element = next( + + varender_postprocessing = f"varenderD{self.vaapi_suffix}postproc" + self._postprocessing_element = next( ( - "vah264dec ! video/x-raw(memory:VAMemory)" + varender_postprocessing for element in elements - if element[1] == "vah264dec" + if element[1] == varender_postprocessing + ), + None, + ) + elif self.gpu_id == 0: + self._compositor_element = next( + ( + "vacompositor" + for element in elements + if element[1] == "vacompositor" ), + None, + ) + + self._encoder_element = next( + ("vah264lpenc" for element in elements if element[1] == "vah264lpenc"), next( - ("decodebin" for element in elements if element[1] == "decodebin"), + ("vah264enc" for element in elements if element[1] == "vah264enc"), None, ), ) - # Postprocessing - # If vaapi_suffix is set, try to find varenderD{suffix}postproc first, e.g. varenderD129postproc for GPU.1 - # If not found, fallback to vapostproc or videoscale - self._postprocessing_element = None - if self.vaapi_suffix is not None: - varender_postprocessing = f"varenderD{self.vaapi_suffix}postproc" - self._postprocessing_element = next( + self._decoder_element = next( ( - varender_postprocessing + "vah264dec ! video/x-raw(memory:VAMemory)" for element in elements - if element[1] == varender_postprocessing + if element[1] == "vah264dec" ), None, ) - if self._postprocessing_element is None: + self._postprocessing_element = next( ("vapostproc" for element in elements if element[1] == "vapostproc"), - next( - ( - "videoscale" - for element in elements - if element[1] == "videoscale" - ), - None, + None, + ) + + if self._compositor_element is None: + self._compositor_element = next( + ("compositor" for element in elements if element[1] == "compositor"), + None, + ) + + if self._encoder_element is None: + self._encoder_element = next( + ( + "x264enc bitrate=16000 speed-preset=superfast" + for element in elements + if element[1] == "x264enc" ), + None, + ) + + if self._decoder_element is None: + self._decoder_element = next( + ("decodebin" for element in elements if element[1] == "decodebin"), + None, + ) + + if self._postprocessing_element is None: + self._postprocessing_element = next( + ("videoscale" for element in elements if element[1] == "videoscale"), + None, ) def compositor_element(self): From 96d65bf64a09f7c72c5aacddd64f7cd8516bf152 Mon Sep 17 00:00:00 2001 From: "Zak, Pawel" Date: Tue, 9 Sep 2025 12:20:44 +0200 Subject: [PATCH 03/13] move to common, adjust simplevs, adjust tests --- .../pipelines/_common/common.py | 151 +++++++++++ .../pipelines/simplevs/pipeline.py | 161 +++++------ .../pipelines/smartnvr/pipeline.py | 252 ++++-------------- .../simplevs_tests/pipeline_test.py | 24 +- .../smartnvr_tests/pipeline_test.py | 89 +++++-- 5 files changed, 369 insertions(+), 308 deletions(-) create mode 100644 tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py new file mode 100644 index 000000000..152765a5d --- /dev/null +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py @@ -0,0 +1,151 @@ +class PipelineElementsSelector: + def __init__(self, parameters: dict, elements: list): + self.parameters = parameters + self.elements = elements + self.gpu_id = -1 + self.vaapi_suffix = None + + # Calculate vaapi_suffix once + device = parameters.get("object_detection_device", "") + + # Determine gpu_id and vaapi_suffix + # If there is only one GPU, device name is just GPU + # If there is more than one GPU, device names are like GPU.0, GPU.1, ... + if device == "GPU": + self.gpu_id = 0 + elif device.startswith("GPU."): + try: + gpu_index = int(device.split(".")[1]) + if gpu_index == 0: + self.gpu_id = 0 + elif gpu_index > 0: + self.vaapi_suffix = str(128 + gpu_index) + self.gpu_id = gpu_index + except (IndexError, ValueError): + self.gpu_id = -1 + else: + self.gpu_id = -1 + + self._compositor_element = None + self._encoder_element = None + self._decoder_element = None + self._postprocessing_element = None + if self.gpu_id > 0: + varender_compositor = f"varenderD{self.vaapi_suffix}compositor" + self._compositor_element = next( + ( + varender_compositor + for element in elements + if element[1] == varender_compositor + ), + None, + ) + + varender_encoder_lp = f"varenderD{self.vaapi_suffix}h264lpenc" + varender_encoder = f"varenderD{self.vaapi_suffix}h264enc" + self._encoder_element = next( + ( + varender_encoder_lp + for element in elements + if element[1] == varender_encoder_lp + ), + next( + ( + varender_encoder + for element in elements + if element[1] == varender_encoder + ), + None, + ), + ) + + varender_decoder = f"varenderD{self.vaapi_suffix}h264dec" + self._decoder_element = next( + ( + f"{varender_decoder} ! video/x-raw(memory:VAMemory)" + for element in elements + if element[1] == varender_decoder + ), + None, + ) + + varender_postprocessing = f"varenderD{self.vaapi_suffix}postproc" + self._postprocessing_element = next( + ( + varender_postprocessing + for element in elements + if element[1] == varender_postprocessing + ), + None, + ) + elif self.gpu_id == 0: + self._compositor_element = next( + ( + "vacompositor" + for element in elements + if element[1] == "vacompositor" + ), + None, + ) + + self._encoder_element = next( + ("vah264lpenc" for element in elements if element[1] == "vah264lpenc"), + next( + ("vah264enc" for element in elements if element[1] == "vah264enc"), + None, + ), + ) + + self._decoder_element = next( + ( + "vah264dec ! video/x-raw(memory:VAMemory)" + for element in elements + if element[1] == "vah264dec" + ), + None, + ) + + self._postprocessing_element = next( + ("vapostproc" for element in elements if element[1] == "vapostproc"), + None, + ) + + if self._compositor_element is None: + self._compositor_element = next( + ("compositor" for element in elements if element[1] == "compositor"), + None, + ) + + if self._encoder_element is None: + self._encoder_element = next( + ( + "x264enc bitrate=16000 speed-preset=superfast" + for element in elements + if element[1] == "x264enc" + ), + None, + ) + + if self._decoder_element is None: + self._decoder_element = next( + ("decodebin" for element in elements if element[1] == "decodebin"), + None, + ) + + if self._postprocessing_element is None: + self._postprocessing_element = next( + ("videoscale" for element in elements if element[1] == "videoscale"), + None, + ) + + def compositor_element(self): + return self._compositor_element + + def encoder_element(self): + return self._encoder_element + + def decoder_element(self): + return self._decoder_element + + def postprocessing_element(self): + return self._postprocessing_element diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py index c94651afc..e6360978d 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py @@ -4,6 +4,7 @@ import struct from gstpipeline import GstPipeline +from pipelines._common.common import PipelineElementsSelector from utils import ( get_video_resolution, UINT8_DTYPE_SIZE, @@ -11,6 +12,8 @@ is_yolov10_model, ) +logger = logging.getLogger("simplevs") + class SimpleVideoStructurizationPipeline(GstPipeline): def __init__(self): @@ -22,39 +25,55 @@ def __init__(self): (330, 110, 445, 170, "Inference", "Object Detection"), ] + # Add shmsink for live-streaming (shared memory) + self._shmsink = ( + "shmsink socket-path=/tmp/shared_memory/video_stream " + "wait-for-connection=false " + "sync=true " + "name=shmsink0 " + ) + + # shmsink branch for live preview (BGR format), width/height will be formatted in evaluate + self._shmsink_branch = ( + "tee name=livetee " + "livetee. ! queue2 ! {encoder} ! h264parse ! mp4mux ! filesink location={VIDEO_OUTPUT_PATH} async=false " + "livetee. ! queue2 ! videoconvert ! video/x-raw,format=BGR,width={output_width},height={output_height} ! {shmsink} " + ) + self._inference_stream_decode_detect_track = ( # Input - "filesrc location={VIDEO_PATH} ! " + "filesrc " + " location={VIDEO_PATH} ! " # Decoder "{decoder} ! " # Detection "gvafpscounter starting-frame=500 ! " "gvadetect " - " {detection_model_config} " - " model-instance-id=detect0 " - " device={object_detection_device} " - " pre-process-backend={object_detection_pre_process_backend} " - " batch-size={object_detection_batch_size} " - " inference-interval={object_detection_inference_interval} " - " {ie_config_parameter} " - " nireq={object_detection_nireq} ! " - "queue ! " + " {detection_model_config} " + " model-instance-id=detect0 " + " pre-process-backend={object_detection_pre_process_backend} " + " device={object_detection_device} " + " batch-size={object_detection_batch_size} " + " inference-interval={object_detection_inference_interval} " + " {ie_config_parameter} " + " nireq={object_detection_nireq} ! " + "queue2 ! " "gvatrack " " tracking-type={tracking_type} ! " - "queue ! " + "queue2 ! " ) self._inference_stream_classify = ( "gvaclassify " - " {classification_model_config} " - " model-instance-id=classify0 " - " device={object_classification_device} " - " pre-process-backend={object_classification_pre_process_backend} " - " batch-size={object_classification_batch_size} " - " inference-interval={object_classification_inference_interval} " - " nireq={object_classification_nireq} " - " reclassify-interval={object_classification_reclassify_interval} ! " - "queue ! " + " {classification_model_config} " + " model-instance-id=classify0 " + " pre-process-backend={object_classification_pre_process_backend} " + " device={object_classification_device} " + " batch-size={object_classification_batch_size} " + " inference-interval={object_classification_inference_interval} " + " nireq={object_classification_nireq} " + " reclassify-interval={object_classification_reclassify_interval} ! " + "queue2 ! " ) self._inference_stream_metadata_processing = ( @@ -68,22 +87,7 @@ def __init__(self): ) self._inference_output_stream = ( - "{encoder} ! h264parse ! mp4mux ! filesink location={VIDEO_OUTPUT_PATH} " - ) - - # Add shmsink for live preview (shared memory) - self._shmsink = ( - "shmsink socket-path=/tmp/shared_memory/video_stream " - "wait-for-connection=false " - "sync=true " - "name=shmsink0 " - ) - - # shmsink branch for live preview (BGR format), width/height will be formatted in evaluate - self._shmsink_branch = ( - "tee name=livetee " - "livetee. ! queue2 ! {encoder} ! h264parse ! mp4mux ! filesink location={VIDEO_OUTPUT_PATH} async=false " - "livetee. ! queue2 ! videoconvert ! video/x-raw,format=BGR,width={width},height={height} ! {shmsink} " + "{encoder} ! h264parse ! mp4mux ! filesink location={VIDEO_OUTPUT_PATH} " ) def evaluate( @@ -97,44 +101,43 @@ def evaluate( if elements is None: elements = [] - # Set decoder element based on device - _decoder_element = ( - "decodebin3 " - if parameters["object_detection_device"] in ["CPU", "NPU"] - else "decodebin3 ! vapostproc ! video/x-raw\\(memory:VAMemory\\)" - ) - - # Set encoder element based on device - _encoder_element = next( - ("vah264enc" for element in elements if element[1] == "vah264enc"), - next( - ("vah264lpenc" for element in elements if element[1] == "vah264lpenc"), - next( - ( - "x264enc bitrate=16000 speed-preset=superfast" - for element in elements - if element[1] == "x264enc" - ), - None, # Fallback to None if no encoder is found - ), - ), - ) - - # Set pre process backed for object detection + # Set pre-process backend for object detection parameters["object_detection_pre_process_backend"] = ( "opencv" if parameters["object_detection_device"] in ["CPU", "NPU"] else "va-surface-sharing" ) - # Set pre process backed for object classification + # Set pre-process backend for object classification parameters["object_classification_pre_process_backend"] = ( "opencv" if parameters["object_classification_device"] in ["CPU", "NPU"] else "va-surface-sharing" ) - # Set model config for object detection + # Use PipelineElementsSelector for element selection + selector = PipelineElementsSelector(parameters, elements) + _compositor_element = selector.compositor_element() + _encoder_element = selector.encoder_element() + _decoder_element = selector.decoder_element() + _postprocessing_element = selector.postprocessing_element() + + # If any of the essential elements is not found, log an error and return an empty string + if not all( + [ + _encoder_element, + _decoder_element, + ] + ): + logger.error("Could not find all necessary elements for the pipeline.") + logger.error(f"Encoder: {_encoder_element}, Decoder: {_decoder_element}") + return "" + else: + logger.info( + f"Using pipeline elements - Encoder: {_encoder_element}, Decoder: {_decoder_element}" + ) + + # Handle object detection parameters and constants detection_model_config = ( f"model={constants['OBJECT_DETECTION_MODEL_PATH']} " f"model-proc={constants['OBJECT_DETECTION_MODEL_PROC']} " @@ -147,29 +150,36 @@ def evaluate( # Set inference config parameter for GPU if using YOLOv10 ie_config_parameter = "" - if parameters["object_detection_device"] == "GPU" and is_yolov10_model( - constants["OBJECT_DETECTION_MODEL_PATH"] - ): + if parameters.get("object_detection_device", "").startswith( + "GPU" + ) and is_yolov10_model(constants["OBJECT_DETECTION_MODEL_PATH"]): ie_config_parameter = "ie-config=GPU_DISABLE_WINOGRAD_CONVOLUTION=YES" - streams = "" - # Prepare shmsink and meta if live_preview_enabled - width = 0 - height = 0 + output_width = 0 + output_height = 0 if parameters["live_preview_enabled"]: # Get resolution using get_video_resolution video_path = constants.get("VIDEO_PATH", "") - width, height = get_video_resolution(video_path) + output_width, output_height = get_video_resolution(video_path) + # Write meta file for live preview try: os.makedirs("/tmp/shared_memory", exist_ok=True) with open(VIDEO_STREAM_META_PATH, "wb") as f: - # height, width, dtype_size=UINT8_DTYPE_SIZE (uint8) - f.write(struct.pack("III", height, width, UINT8_DTYPE_SIZE)) + # height=output_height, width=output_width, dtype_size=UINT8_DTYPE_SIZE (uint8) + f.write( + struct.pack( + "III", output_height, output_width, UINT8_DTYPE_SIZE + ) + ) except Exception as e: - logging.warning(f"Could not write shared memory meta file: {e}") + logger.warning(f"Could not write shared memory meta file: {e}") + + # Create the streams + streams = "" + # Handle inference channels for i in range(inference_channels): streams += self._inference_stream_decode_detect_track.format( **parameters, @@ -225,8 +235,8 @@ def evaluate( streams += self._shmsink_branch.format( **constants, encoder=_encoder_element, - width=width, - height=height, + output_width=output_width, + output_height=output_height, shmsink=self._shmsink, ) else: @@ -236,4 +246,5 @@ def evaluate( else: streams += "fakesink " + # Evaluate the pipeline return "gst-launch-1.0 -q " + streams diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py index 2f03e4ee4..25f299561 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py @@ -5,6 +5,7 @@ import struct from gstpipeline import GstPipeline +from pipelines._common.common import PipelineElementsSelector from utils import UINT8_DTYPE_SIZE, VIDEO_STREAM_META_PATH, is_yolov10_model logger = logging.getLogger("smartnvr") @@ -67,6 +68,7 @@ def __init__(self): ) self._inference_stream_decode_detect_track = ( + # Input "filesrc " " location={VIDEO_PATH} ! " "qtdemux ! " @@ -78,7 +80,9 @@ def __init__(self): " location=/tmp/stream{id}.mp4 " "t{id}. ! " "queue2 ! " + # Decoder "{decoder} ! " + # Detection "gvafpscounter starting-frame=500 ! " "gvadetect " " {detection_model_config} " @@ -153,17 +157,6 @@ def evaluate( else "va-surface-sharing" ) - # Compute total number of channels - channels = regular_channels + inference_channels - - # Create a sink for each channel - sinks = "" - grid_size = math.ceil(math.sqrt(channels)) - for i in range(channels): - xpos = 640 * (i % grid_size) - ypos = 360 * (i // grid_size) - sinks += self._sink.format(id=i, xpos=xpos, ypos=ypos) - # Use PipelineElementsSelector for element selection selector = PipelineElementsSelector(parameters, elements) _compositor_element = selector.compositor_element() @@ -193,26 +186,60 @@ def evaluate( # Create the streams streams = "" - # Handle inference channels - for i in range(inference_channels): - # Handle object detection parameters and constants + # Handle object detection parameters and constants + detection_model_config = ( + f"model={constants['OBJECT_DETECTION_MODEL_PATH']} " + f"model-proc={constants['OBJECT_DETECTION_MODEL_PROC']} " + ) + + if not constants["OBJECT_DETECTION_MODEL_PROC"]: detection_model_config = ( f"model={constants['OBJECT_DETECTION_MODEL_PATH']} " - f"model-proc={constants['OBJECT_DETECTION_MODEL_PROC']} " ) - if not constants["OBJECT_DETECTION_MODEL_PROC"]: - detection_model_config = ( - f"model={constants['OBJECT_DETECTION_MODEL_PATH']} " - ) + # Set inference config parameter for GPU if using YOLOv10 + ie_config_parameter = "" + if parameters.get("object_detection_device", "").startswith( + "GPU" + ) and is_yolov10_model(constants["OBJECT_DETECTION_MODEL_PATH"]): + ie_config_parameter = "ie-config=GPU_DISABLE_WINOGRAD_CONVOLUTION=YES" + + # Compute total number of channels + channels = regular_channels + inference_channels + + # Create a sink for each channel + sinks = "" + grid_size = math.ceil(math.sqrt(channels)) + for i in range(channels): + xpos = 640 * (i % grid_size) + ypos = 360 * (i // grid_size) + sinks += self._sink.format(id=i, xpos=xpos, ypos=ypos) + + # Prepare shmsink and meta if live_preview_enabled + output_width = 0 + output_height = 0 + if parameters["live_preview_enabled"]: + # Calculate output video size for grid layout to ensure same resolution for shmsink and output file + output_width = 640 * grid_size + output_height = 360 * ( + (channels + grid_size - 1) // grid_size + ) # ceil(channels / grid_size) - # Set inference config parameter for GPU if using YOLOv10 - ie_config_parameter = "" - if parameters.get("object_detection_device", "").startswith( - "GPU" - ) and is_yolov10_model(constants["OBJECT_DETECTION_MODEL_PATH"]): - ie_config_parameter = "ie-config=GPU_DISABLE_WINOGRAD_CONVOLUTION=YES" + # Write meta file for live preview + try: + os.makedirs("/tmp/shared_memory", exist_ok=True) + with open(VIDEO_STREAM_META_PATH, "wb") as f: + # height=output_height, width=output_width, dtype_size=UINT8_DTYPE_SIZE (uint8) + f.write( + struct.pack( + "III", output_height, output_width, UINT8_DTYPE_SIZE + ) + ) + except Exception as e: + logger.warning(f"Could not write shared memory meta file: {e}") + # Handle inference channels + for i in range(inference_channels): streams += self._inference_stream_decode_detect_track.format( **parameters, **constants, @@ -228,6 +255,7 @@ def evaluate( constants["OBJECT_CLASSIFICATION_MODEL_PATH"] == "Disabled" or parameters["object_classification_device"] == "Disabled" ): + # Set model config for object classification classification_model_config = ( f"model={constants['OBJECT_CLASSIFICATION_MODEL_PATH']} " f"model-proc={constants['OBJECT_CLASSIFICATION_MODEL_PROC']} " @@ -245,10 +273,11 @@ def evaluate( classification_model_config=classification_model_config, ) - # Overlay inference results on the inferenced video if enabled + # Overlay inference results on the inferred video if enabled if parameters["pipeline_watermark_enabled"]: streams += "gvawatermark ! " + # Metadata processing and publishing streams += self._inference_stream_metadata_processing.format( **parameters, **constants, @@ -282,25 +311,7 @@ def evaluate( ) # Compose pipeline depending on live_preview_enabled if parameters["live_preview_enabled"]: - # Calculate output video size for grid layout to ensure same resolution for shmsink and output file - output_width = 640 * grid_size - output_height = 360 * ( - (channels + grid_size - 1) // grid_size - ) # ceil(channels / grid_size) - # Always produce both file and live stream outputs - try: - os.makedirs("/tmp/shared_memory", exist_ok=True) - with open(VIDEO_STREAM_META_PATH, "wb") as f: - # width=output_height, height=output_width, dtype_size=UINT8_DTYPE_SIZE (uint8) - f.write( - struct.pack( - "III", output_height, output_width, UINT8_DTYPE_SIZE - ) - ) - except Exception as e: - logger.warning(f"Could not write shared memory meta file: {e}") - streams = ( self._compositor_with_tee.format( **constants, @@ -327,156 +338,3 @@ def evaluate( # Evaluate the pipeline return "gst-launch-1.0 -q " + streams - - -class PipelineElementsSelector: - def __init__(self, parameters: dict, elements: list): - self.parameters = parameters - self.elements = elements - self.gpu_id = -1 - self.vaapi_suffix = None - - # Calculate vaapi_suffix once - device = parameters.get("object_detection_device", "") - - # Determine gpu_id and vaapi_suffix - # If there is only one GPU, device name is just GPU - # If there is more than one GPU, device names are like GPU.0, GPU.1, ... - if device == "GPU": - self.gpu_id = 0 - elif device.startswith("GPU."): - try: - gpu_index = int(device.split(".")[1]) - if gpu_index == 0: - self.gpu_id = 0 - elif gpu_index > 0: - self.vaapi_suffix = str(128 + gpu_index) - self.gpu_id = gpu_index - except (IndexError, ValueError): - self.gpu_id = -1 - else: - self.gpu_id = -1 - - self._compositor_element = None - self._encoder_element = None - self._decoder_element = None - self._postprocessing_element = None - if self.gpu_id > 0: - varender_compositor = f"varenderD{self.vaapi_suffix}compositor" - self._compositor_element = next( - ( - varender_compositor - for element in elements - if element[1] == varender_compositor - ), - None, - ) - - varender_encoder_lp = f"varenderD{self.vaapi_suffix}h264lpenc" - varender_encoder = f"varenderD{self.vaapi_suffix}h264enc" - self._encoder_element = next( - ( - varender_encoder_lp - for element in elements - if element[1] == varender_encoder_lp - ), - next( - ( - varender_encoder - for element in elements - if element[1] == varender_encoder - ), - None, - ), - ) - - varender_decoder = f"varenderD{self.vaapi_suffix}h264dec" - self._decoder_element = next( - ( - f"{varender_decoder} ! video/x-raw(memory:VAMemory)" - for element in elements - if element[1] == varender_decoder - ), - None, - ) - - varender_postprocessing = f"varenderD{self.vaapi_suffix}postproc" - self._postprocessing_element = next( - ( - varender_postprocessing - for element in elements - if element[1] == varender_postprocessing - ), - None, - ) - elif self.gpu_id == 0: - self._compositor_element = next( - ( - "vacompositor" - for element in elements - if element[1] == "vacompositor" - ), - None, - ) - - self._encoder_element = next( - ("vah264lpenc" for element in elements if element[1] == "vah264lpenc"), - next( - ("vah264enc" for element in elements if element[1] == "vah264enc"), - None, - ), - ) - - self._decoder_element = next( - ( - "vah264dec ! video/x-raw(memory:VAMemory)" - for element in elements - if element[1] == "vah264dec" - ), - None, - ) - - self._postprocessing_element = next( - ("vapostproc" for element in elements if element[1] == "vapostproc"), - None, - ) - - if self._compositor_element is None: - self._compositor_element = next( - ("compositor" for element in elements if element[1] == "compositor"), - None, - ) - - if self._encoder_element is None: - self._encoder_element = next( - ( - "x264enc bitrate=16000 speed-preset=superfast" - for element in elements - if element[1] == "x264enc" - ), - None, - ) - - if self._decoder_element is None: - self._decoder_element = next( - ("decodebin" for element in elements if element[1] == "decodebin"), - None, - ) - - if self._postprocessing_element is None: - self._postprocessing_element = next( - ("videoscale" for element in elements if element[1] == "videoscale"), - None, - ) - - def compositor_element(self): - return self._compositor_element - - def encoder_element(self): - return self._encoder_element - - def decoder_element(self): - return self._decoder_element - - def postprocessing_element(self): - return self._postprocessing_element diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py index 5fe7b181f..39cb4d33c 100755 --- a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py @@ -62,10 +62,10 @@ def test_evaluate_cpu(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "decodebin3", "..."), - ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), - ("va", "vapostproc", "..."), + ("va", "compositor", "..."), + ("va", "x264enc", "..."), + ("va", "decodebin", "..."), + ("va", "videoscale", "..."), ], ) @@ -77,7 +77,7 @@ def test_evaluate_cpu(self): self.assertIn("model-proc=classification_model_proc.json", result) # Check that the decoder element is correctly used - self.assertIn("decodebin3", result) + self.assertIn("decodebin", result) # Check that opencv is used for pre-processing self.assertIn("pre-process-backend=opencv", result) @@ -110,7 +110,7 @@ def test_evaluate_gpu(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "decodebin3", "..."), + ("va", "vacompositor", "..."), ("va", "vah264enc", "..."), ("va", "vah264dec", "..."), ("va", "vapostproc", "..."), @@ -125,9 +125,7 @@ def test_evaluate_gpu(self): self.assertIn("model-proc=classification_model_proc.json", result) # Check that the decoder element is correctly used - self.assertIn( - "decodebin3 ! vapostproc ! video/x-raw\\(memory:VAMemory\\)", result - ) + self.assertIn("vah264dec", result) # Check that va-surface-sharing is used for pre-processing self.assertIn("pre-process-backend=va-surface-sharing", result) @@ -163,7 +161,7 @@ def test_evaluate_no_model_proc(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "decodebin3", "..."), + ("va", "vacompositor", "..."), ("va", "vah264enc", "..."), ("va", "vah264dec", "..."), ("va", "vapostproc", "..."), @@ -200,7 +198,7 @@ def test_evaluate_no_overlay(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "decodebin3", "..."), + ("va", "vah264dec", "..."), ("va", "vah264enc", "..."), ], ) @@ -235,8 +233,8 @@ def test_evaluate_no_overlay_no_video(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "decodebin3", "..."), ("va", "vah264enc", "..."), + ("va", "vah264dec", "..."), ], ) @@ -270,8 +268,8 @@ def test_evaluate_no_overlay_when_video_disabled(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "decodebin3", "..."), ("va", "vah264enc", "..."), + ("va", "vah264dec", "..."), ], ) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py index d4ee7d71b..4e7a8784e 100755 --- a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py @@ -73,10 +73,10 @@ def test_evaluate_no_overlay(self): regular_channels=self.regular_channels, inference_channels=self.inference_channels, elements=[ - ("va", "vacompositor", "..."), - ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), - ("va", "vapostproc", "..."), + ("va", "compositor", "..."), + ("va", "x264enc", "..."), + ("va", "decodebin", "..."), + ("va", "videoscale", "..."), ], ) @@ -109,10 +109,10 @@ def test_evaluate_classification_disabled_by_device(self): regular_channels=self.regular_channels, inference_channels=self.inference_channels, elements=[ - ("va", "vacompositor", "..."), - ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), - ("va", "vapostproc", "..."), + ("va", "compositor", "..."), + ("va", "x264enc", "..."), + ("va", "decodebin", "..."), + ("va", "videoscale", "..."), ], ) @@ -149,10 +149,10 @@ def test_evaluate_classification_disabled_by_model(self): regular_channels=self.regular_channels, inference_channels=self.inference_channels, elements=[ - ("va", "vacompositor", "..."), - ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), - ("va", "vapostproc", "..."), + ("va", "compositor", "..."), + ("va", "x264enc", "..."), + ("va", "decodebin", "..."), + ("va", "videoscale", "..."), ], ) @@ -182,10 +182,10 @@ def test_evaluate_cpu(self): regular_channels=self.regular_channels, inference_channels=self.inference_channels, elements=[ - ("va", "vacompositor", "..."), - ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), - ("va", "vapostproc", "..."), + ("va", "compositor", "..."), + ("va", "x264enc", "..."), + ("va", "decodebin", "..."), + ("va", "videoscale", "..."), ], ) @@ -202,15 +202,15 @@ def test_evaluate_cpu(self): # Check that opencv is used for pre-processing self.assertIn("pre-process-backend=opencv", result) - def test_evaluate_gpu(self): + def test_evaluate_gpu_0(self): result = self.pipeline.evaluate( constants=self.constants, parameters={ - "object_detection_device": "GPU.1", + "object_detection_device": "GPU.0", "object_detection_batch_size": 0, "object_detection_inference_interval": 1, "object_detection_nireq": 0, - "object_classification_device": "GPU.1", + "object_classification_device": "GPU.0", "object_classification_batch_size": 0, "object_classification_inference_interval": 1, "object_classification_nireq": 0, @@ -242,6 +242,49 @@ def test_evaluate_gpu(self): # Check that va-surface-sharing is used for pre-processing self.assertIn("pre-process-backend=va-surface-sharing", result) + # Check that the right vaapi elements are used + self.assertIn("vacompositor", result) + + def test_evaluate_gpu_1(self): + result = self.pipeline.evaluate( + constants=self.constants, + parameters={ + "object_detection_device": "GPU.1", + "object_detection_batch_size": 0, + "object_detection_inference_interval": 1, + "object_detection_nireq": 0, + "object_classification_device": "GPU.1", + "object_classification_batch_size": 0, + "object_classification_inference_interval": 1, + "object_classification_nireq": 0, + "object_classification_reclassify_interval": 1, + "tracking_type": "short-term-imageless", + "pipeline_watermark_enabled": True, + "live_preview_enabled": False, + }, + regular_channels=self.regular_channels, + inference_channels=self.inference_channels, + elements=[ + ("va", "varenderD129compositor", "..."), + ("va", "varenderD129h264enc", "..."), + ("va", "varenderD129h264dec", "..."), + ("va", "varenderD129postproc", "..."), + ], + ) + + # Common checks + self.common_checks(result) + self.output_present_check(result) + self.sink_count_check(result) + self.gvaclassify_count_check(result, self.inference_channels) + + # Check that model proc is used + self.assertIn("model-proc=detection_model_proc.json", result) + self.assertIn("model-proc=classification_model_proc.json", result) + + # Check that va-surface-sharing is used for pre-processing + self.assertIn("pre-process-backend=va-surface-sharing", result) + # Check that the right vaapi elements are used self.assertIn("varenderD129compositor", result) @@ -272,10 +315,10 @@ def test_evaluate_no_model_proc(self): regular_channels=self.regular_channels, inference_channels=self.inference_channels, elements=[ - ("va", "vacompositor", "..."), - ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), - ("va", "vapostproc", "..."), + ("va", "compositor", "..."), + ("va", "x264enc", "..."), + ("va", "decodebin", "..."), + ("va", "videoscale", "..."), ], ) From b914c3e7710e5174caaf64c120f5794d9b9b5433 Mon Sep 17 00:00:00 2001 From: "Zak, Pawel" Date: Wed, 10 Sep 2025 13:47:05 +0200 Subject: [PATCH 04/13] corrections --- .../pipelines/_common/common.py | 2 +- .../pipelines/simplevs/pipeline.py | 4 ++-- .../pipelines/smartnvr/pipeline.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py index 152765a5d..79dc1cc69 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py @@ -128,7 +128,7 @@ def __init__(self, parameters: dict, elements: list): if self._decoder_element is None: self._decoder_element = next( - ("decodebin" for element in elements if element[1] == "decodebin"), + ("decodebin3" for element in elements if element[1] == "decodebin3"), None, ) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py index e6360978d..26ed62e0f 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py @@ -44,6 +44,8 @@ def __init__(self): # Input "filesrc " " location={VIDEO_PATH} ! " + "qtdemux ! " + "h264parse ! " # Decoder "{decoder} ! " # Detection @@ -117,10 +119,8 @@ def evaluate( # Use PipelineElementsSelector for element selection selector = PipelineElementsSelector(parameters, elements) - _compositor_element = selector.compositor_element() _encoder_element = selector.encoder_element() _decoder_element = selector.decoder_element() - _postprocessing_element = selector.postprocessing_element() # If any of the essential elements is not found, log an error and return an empty string if not all( diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py index 25f299561..90eaca5c6 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py @@ -299,7 +299,6 @@ def evaluate( **constants, id=i, decoder=_decoder_element, - postprocessing=_postprocessing_element, ) # sink to compositor or fake sink depending on the compose flag streams += self._sink_to_compositor.format( From d4de56e5f050e49c1a33c0611def63807f290ddb Mon Sep 17 00:00:00 2001 From: "Zak, Pawel" Date: Wed, 17 Sep 2025 13:54:18 +0200 Subject: [PATCH 05/13] introduce PipelineElementSelectionInstructions --- .../pipelines/_common/common.py | 213 +++++++----------- .../pipelines/smartnvr/pipeline.py | 80 ++++++- 2 files changed, 149 insertions(+), 144 deletions(-) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py index 79dc1cc69..69e99f3e1 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py @@ -1,151 +1,90 @@ -class PipelineElementsSelector: - def __init__(self, parameters: dict, elements: list): - self.parameters = parameters - self.elements = elements - self.gpu_id = -1 - self.vaapi_suffix = None +from typing import List, Tuple, Dict, Optional + +# Keys for device selection +GPU_0 = "GPU_0" +GPU_N = "GPU_N" +OTHER = "OTHER" +# Placeholder for vaapi_suffix to be replaced at runtime +VAAPI_SUFFIX_PLACEHOLDER = "{vaapi_suffix}" + + +class PipelineElementSelectionInstructions: + def __init__( + self, + compositor: Optional[Dict[str, List[Tuple[str, str]]]] = None, + encoder: Optional[Dict[str, List[Tuple[str, str]]]] = None, + decoder: Optional[Dict[str, List[Tuple[str, str]]]] = None, + postprocessing: Optional[Dict[str, List[Tuple[str, str]]]] = None, + ): + # Each field is a dict: key is GPU_0, GPU_N, or OTHER, value is list of (search, result) + self.compositor = compositor or {} + self.encoder = encoder or {} + self.decoder = decoder or {} + self.postprocessing = postprocessing or {} + - # Calculate vaapi_suffix once +class PipelineElementsSelector: + def __init__( + self, + selection_instructions: PipelineElementSelectionInstructions, + ): + self.instructions = selection_instructions + + def select_elements( + self, + parameters: dict, + elements: list, + ) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: + gpu_id = -1 + vaapi_suffix = None device = parameters.get("object_detection_device", "") # Determine gpu_id and vaapi_suffix # If there is only one GPU, device name is just GPU # If there is more than one GPU, device names are like GPU.0, GPU.1, ... if device == "GPU": - self.gpu_id = 0 + gpu_id = 0 elif device.startswith("GPU."): try: gpu_index = int(device.split(".")[1]) if gpu_index == 0: - self.gpu_id = 0 + gpu_id = 0 elif gpu_index > 0: - self.vaapi_suffix = str(128 + gpu_index) - self.gpu_id = gpu_index + vaapi_suffix = str(128 + gpu_index) + gpu_id = gpu_index except (IndexError, ValueError): - self.gpu_id = -1 + gpu_id = -1 else: - self.gpu_id = -1 - - self._compositor_element = None - self._encoder_element = None - self._decoder_element = None - self._postprocessing_element = None - if self.gpu_id > 0: - varender_compositor = f"varenderD{self.vaapi_suffix}compositor" - self._compositor_element = next( - ( - varender_compositor - for element in elements - if element[1] == varender_compositor - ), - None, - ) - - varender_encoder_lp = f"varenderD{self.vaapi_suffix}h264lpenc" - varender_encoder = f"varenderD{self.vaapi_suffix}h264enc" - self._encoder_element = next( - ( - varender_encoder_lp - for element in elements - if element[1] == varender_encoder_lp - ), - next( - ( - varender_encoder - for element in elements - if element[1] == varender_encoder - ), - None, - ), - ) - - varender_decoder = f"varenderD{self.vaapi_suffix}h264dec" - self._decoder_element = next( - ( - f"{varender_decoder} ! video/x-raw(memory:VAMemory)" - for element in elements - if element[1] == varender_decoder - ), - None, - ) - - varender_postprocessing = f"varenderD{self.vaapi_suffix}postproc" - self._postprocessing_element = next( - ( - varender_postprocessing - for element in elements - if element[1] == varender_postprocessing - ), - None, - ) - elif self.gpu_id == 0: - self._compositor_element = next( - ( - "vacompositor" - for element in elements - if element[1] == "vacompositor" - ), - None, - ) - - self._encoder_element = next( - ("vah264lpenc" for element in elements if element[1] == "vah264lpenc"), - next( - ("vah264enc" for element in elements if element[1] == "vah264enc"), - None, - ), - ) - - self._decoder_element = next( - ( - "vah264dec ! video/x-raw(memory:VAMemory)" - for element in elements - if element[1] == "vah264dec" - ), - None, - ) - - self._postprocessing_element = next( - ("vapostproc" for element in elements if element[1] == "vapostproc"), - None, - ) - - if self._compositor_element is None: - self._compositor_element = next( - ("compositor" for element in elements if element[1] == "compositor"), - None, - ) - - if self._encoder_element is None: - self._encoder_element = next( - ( - "x264enc bitrate=16000 speed-preset=superfast" - for element in elements - if element[1] == "x264enc" - ), - None, - ) - - if self._decoder_element is None: - self._decoder_element = next( - ("decodebin3" for element in elements if element[1] == "decodebin3"), - None, - ) - - if self._postprocessing_element is None: - self._postprocessing_element = next( - ("videoscale" for element in elements if element[1] == "videoscale"), - None, - ) - - def compositor_element(self): - return self._compositor_element - - def encoder_element(self): - return self._encoder_element - - def decoder_element(self): - return self._decoder_element - - def postprocessing_element(self): - return self._postprocessing_element + gpu_id = -1 + + def _select_element(field_dict: Dict[str, List[Tuple[str, str]]]) -> Optional[str]: + key = OTHER + if gpu_id == 0: + key = GPU_0 + elif gpu_id > 0: + key = GPU_N + + pairs = field_dict.get(key, []) + # Add OTHER pairs as fallback if key is not OTHER + if key != OTHER: + pairs = pairs + field_dict.get(OTHER, []) + + if not pairs: + return None + + for search, result in pairs: + if VAAPI_SUFFIX_PLACEHOLDER in search or VAAPI_SUFFIX_PLACEHOLDER in result: + suffix = vaapi_suffix if vaapi_suffix is not None else "" + search = search.replace(VAAPI_SUFFIX_PLACEHOLDER, suffix) + result = result.replace(VAAPI_SUFFIX_PLACEHOLDER, suffix) + for element in elements: + if element[1] == search: + return result + return None + + compositor_element = _select_element(self.instructions.compositor) + encoder_element = _select_element(self.instructions.encoder) + decoder_element = _select_element(self.instructions.decoder) + postprocessing_element = _select_element(self.instructions.postprocessing) + + return compositor_element, encoder_element, decoder_element, postprocessing_element diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py index 90eaca5c6..e27d21661 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py @@ -5,12 +5,11 @@ import struct from gstpipeline import GstPipeline -from pipelines._common.common import PipelineElementsSelector +from pipelines._common.common import PipelineElementsSelector, PipelineElementSelectionInstructions, VAAPI_SUFFIX_PLACEHOLDER, GPU_0, GPU_N,OTHER from utils import UINT8_DTYPE_SIZE, VIDEO_STREAM_META_PATH, is_yolov10_model logger = logging.getLogger("smartnvr") - class SmartNVRPipeline(GstPipeline): def __init__(self): super().__init__() @@ -132,6 +131,72 @@ def __init__(self): "comp.sink_{id} " ) + self._selector = PipelineElementsSelector( + PipelineElementSelectionInstructions( + compositor={ + GPU_0: [ + ("vacompositor", "vacompositor"), + ], + GPU_N: [ + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}compositor", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}compositor", + ), + ], + OTHER: [ + ("compositor", "compositor"), + ], + }, + encoder={ + GPU_0: [ + ("vah264lpenc", "vah264lpenc"), + ("vah264enc", "vah264enc"), + ], + GPU_N: [ + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264lpenc", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264lpenc", + ), + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264enc", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264enc", + ), + ], + OTHER: [ + ("x264enc", "x264enc bitrate=16000 speed-preset=superfast"), + ], + }, + decoder={ + GPU_0: [ + ("vah264dec", "vah264dec ! video/x-raw(memory:VAMemory)"), + ], + GPU_N: [ + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264dec", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264dec ! video/x-raw(memory:VAMemory)", + ), + ], + OTHER: [ + ("decodebin", "decodebin"), + ], + }, + postprocessing={ + GPU_0: [ + ("vapostproc", "vapostproc"), + ], + GPU_N: [ + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}postproc", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}postproc", + ), + ], + OTHER: [ + ("videoscale", "videoscale"), + ], + }, + ) + ) + def evaluate( self, constants: dict, @@ -158,11 +223,12 @@ def evaluate( ) # Use PipelineElementsSelector for element selection - selector = PipelineElementsSelector(parameters, elements) - _compositor_element = selector.compositor_element() - _encoder_element = selector.encoder_element() - _decoder_element = selector.decoder_element() - _postprocessing_element = selector.postprocessing_element() + ( + _compositor_element, + _encoder_element, + _decoder_element, + _postprocessing_element, + ) = self._selector.select_elements(parameters, elements) # If any of the essential elements is not found, log an error and return an empty string if not all( From ec26cb8d5b15131c566268533bdf3e97fd89449b Mon Sep 17 00:00:00 2001 From: "Zak, Pawel" Date: Wed, 17 Sep 2025 15:43:47 +0200 Subject: [PATCH 06/13] simplevs --- .../pipelines/simplevs/pipeline.py | 56 ++++++++++++++----- .../pipelines/smartnvr/pipeline.py | 6 +- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py index 26ed62e0f..25f4ea73c 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py @@ -4,17 +4,11 @@ import struct from gstpipeline import GstPipeline -from pipelines._common.common import PipelineElementsSelector -from utils import ( - get_video_resolution, - UINT8_DTYPE_SIZE, - VIDEO_STREAM_META_PATH, - is_yolov10_model, -) +from pipelines._common.common import PipelineElementsSelector, PipelineElementSelectionInstructions, VAAPI_SUFFIX_PLACEHOLDER, GPU_0, GPU_N,OTHER +from utils import get_video_resolution, UINT8_DTYPE_SIZE, VIDEO_STREAM_META_PATH, is_yolov10_model logger = logging.getLogger("simplevs") - class SimpleVideoStructurizationPipeline(GstPipeline): def __init__(self): super().__init__() @@ -44,8 +38,6 @@ def __init__(self): # Input "filesrc " " location={VIDEO_PATH} ! " - "qtdemux ! " - "h264parse ! " # Decoder "{decoder} ! " # Detection @@ -92,6 +84,41 @@ def __init__(self): "{encoder} ! h264parse ! mp4mux ! filesink location={VIDEO_OUTPUT_PATH} " ) + self._selector = PipelineElementsSelector( + PipelineElementSelectionInstructions( + encoder={ + GPU_0: [ + ("vah264lpenc", "vah264lpenc"), + ("vah264enc", "vah264enc"), + ], + GPU_N: [ + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264lpenc", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264lpenc", + ), + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264enc", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264enc", + ), + ], + OTHER: [ + ("x264enc", "x264enc bitrate=16000 speed-preset=superfast"), + ], + }, + decoder={ + GPU_0: [ + ("decodebin3", "decodebin3 ! vapostproc ! video/x-raw\\(memory:VAMemory\\)"), + ], + GPU_N: [ + ("decodebin3", "decodebin3 ! vapostproc ! video/x-raw\\(memory:VAMemory\\)"), + ], + OTHER: [ + ("decodebin3", "decodebin3"), + ], + }, + ) + ) + def evaluate( self, constants: dict, @@ -118,9 +145,12 @@ def evaluate( ) # Use PipelineElementsSelector for element selection - selector = PipelineElementsSelector(parameters, elements) - _encoder_element = selector.encoder_element() - _decoder_element = selector.decoder_element() + ( + _, + _encoder_element, + _decoder_element, + _, + ) = self._selector.select_elements(parameters, elements) # If any of the essential elements is not found, log an error and return an empty string if not all( diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py index e27d21661..01b176e0d 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py @@ -249,9 +249,6 @@ def evaluate( f"Using pipeline elements - Compositor: {_compositor_element}, Encoder: {_encoder_element}, Decoder: {_decoder_element}, Postprocessing: {_postprocessing_element}" ) - # Create the streams - streams = "" - # Handle object detection parameters and constants detection_model_config = ( f"model={constants['OBJECT_DETECTION_MODEL_PATH']} " @@ -304,6 +301,9 @@ def evaluate( except Exception as e: logger.warning(f"Could not write shared memory meta file: {e}") + # Create the streams + streams = "" + # Handle inference channels for i in range(inference_channels): streams += self._inference_stream_decode_detect_track.format( From 6620fe34ea7e53e87192f81eda22ea7cf8e983cf Mon Sep 17 00:00:00 2001 From: "Zak, Pawel" Date: Thu, 18 Sep 2025 09:28:52 +0200 Subject: [PATCH 07/13] fix decodes --- .../pipelines/_common/common.py | 3 ++ .../pipelines/simplevs/pipeline.py | 38 ++++++++++++++++--- .../pipelines/smartnvr/pipeline.py | 6 +-- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py index 69e99f3e1..5135b993e 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py @@ -73,6 +73,9 @@ def _select_element(field_dict: Dict[str, List[Tuple[str, str]]]) -> Optional[st return None for search, result in pairs: + if search == "": # to support optional parameters + return result + if VAAPI_SUFFIX_PLACEHOLDER in search or VAAPI_SUFFIX_PLACEHOLDER in result: suffix = vaapi_suffix if vaapi_suffix is not None else "" search = search.replace(VAAPI_SUFFIX_PLACEHOLDER, suffix) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py index 25f4ea73c..1cb0b3312 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py @@ -38,8 +38,11 @@ def __init__(self): # Input "filesrc " " location={VIDEO_PATH} ! " + "qtdemux ! " + "h264parse ! " # Decoder "{decoder} ! " + "{postprocessing}" # postprocessing is optional, if present it will have a trailing " ! " # Detection "gvafpscounter starting-frame=500 ! " "gvadetect " @@ -107,15 +110,38 @@ def __init__(self): }, decoder={ GPU_0: [ - ("decodebin3", "decodebin3 ! vapostproc ! video/x-raw\\(memory:VAMemory\\)"), + ( + "vaapidecodebin", + "vaapidecodebin", + ), ], GPU_N: [ - ("decodebin3", "decodebin3 ! vapostproc ! video/x-raw\\(memory:VAMemory\\)"), + ( + "vaapidecodebin", + "vaapidecodebin", + ), ], OTHER: [ ("decodebin3", "decodebin3"), ], }, + postprocessing={ + GPU_0: [ + ( + "vapostproc", + "vapostproc ! video/x-raw\\(memory:VAMemory\\) ! ", + ), + ], + GPU_N: [ + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}postproc", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}postproc ! video/x-raw\\(memory:VAMemory\\) ! ", + ), + ], + OTHER: [ + ("", ""), # force empty string if no postprocessing is needed + ], + }, ) ) @@ -149,10 +175,11 @@ def evaluate( _, _encoder_element, _decoder_element, - _, + _postprocessing_element, ) = self._selector.select_elements(parameters, elements) # If any of the essential elements is not found, log an error and return an empty string + # _postprocessing_element is optional, so don't include it in the check if not all( [ _encoder_element, @@ -160,11 +187,11 @@ def evaluate( ] ): logger.error("Could not find all necessary elements for the pipeline.") - logger.error(f"Encoder: {_encoder_element}, Decoder: {_decoder_element}") + logger.error(f"Encoder: {_encoder_element}, Decoder: {_decoder_element}, Postprocessing: {_postprocessing_element}") return "" else: logger.info( - f"Using pipeline elements - Encoder: {_encoder_element}, Decoder: {_decoder_element}" + f"Using pipeline elements - Encoder: {_encoder_element}, Decoder: {_decoder_element}, Postprocessing: {_postprocessing_element}" ) # Handle object detection parameters and constants @@ -215,6 +242,7 @@ def evaluate( **parameters, **constants, decoder=_decoder_element, + postprocessing=_postprocessing_element, detection_model_config=detection_model_config, ie_config_parameter=ie_config_parameter, ) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py index 01b176e0d..ac5b046e6 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py @@ -168,16 +168,16 @@ def __init__(self): }, decoder={ GPU_0: [ - ("vah264dec", "vah264dec ! video/x-raw(memory:VAMemory)"), + ("vah264dec", "vah264dec ! video/x-raw\\(memory:VAMemory\\)"), ], GPU_N: [ ( f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264dec", - f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264dec ! video/x-raw(memory:VAMemory)", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264dec ! video/x-raw\\(memory:VAMemory\\)", ), ], OTHER: [ - ("decodebin", "decodebin"), + ("decodebin3", "decodebin3"), ], }, postprocessing={ From b6334a60e6621227dee31e01e7abeb6d05cfdf47 Mon Sep 17 00:00:00 2001 From: "Zak, Pawel" Date: Thu, 18 Sep 2025 09:55:29 +0200 Subject: [PATCH 08/13] decodebin seems to be faster and less problematic for CPU --- .../pipelines/simplevs/pipeline.py | 2 +- .../pipelines/smartnvr/pipeline.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py index 1cb0b3312..b901afabb 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py @@ -122,7 +122,7 @@ def __init__(self): ), ], OTHER: [ - ("decodebin3", "decodebin3"), + ("decodebin", "decodebin"), ], }, postprocessing={ diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py index ac5b046e6..df59139d1 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py @@ -177,7 +177,7 @@ def __init__(self): ), ], OTHER: [ - ("decodebin3", "decodebin3"), + ("decodebin", "decodebin"), ], }, postprocessing={ From e3009b7497936cb9d330654ad6c37742529507e1 Mon Sep 17 00:00:00 2001 From: "Zak, Pawel" Date: Thu, 18 Sep 2025 11:21:56 +0200 Subject: [PATCH 09/13] add compositor_device setting --- .../app.py | 19 ++++++ .../pipelines/_common/common.py | 65 ++++++++++--------- .../pipelines/simplevs/config.yaml | 1 + .../pipelines/smartnvr/config.yaml | 1 + .../utils.py | 3 + 5 files changed, 60 insertions(+), 29 deletions(-) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/app.py b/tools/visual-pipeline-and-platform-evaluation-tool/app.py index fed4dfee0..880778247 100755 --- a/tools/visual-pipeline-and-platform-evaluation-tool/app.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/app.py @@ -796,6 +796,14 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool" elem_id="object_classification_reclassify_interval", ) + # Compositor device + compositor_device = gr.Dropdown( + label="Compositor Device", + choices=devices, + value=preferred_device, + elem_id="compositor_device", + ) + pipeline_watermark_enabled = gr.Checkbox( label="Overlay inference results on inference channels", value=True, @@ -864,6 +872,7 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool" components.add(object_classification_inference_interval) components.add(object_classification_nireq) components.add(object_classification_reclassify_interval) + components.add(compositor_device) components.add(pipeline_watermark_enabled) components.add(pipeline_video_enabled) components.add(live_preview_enabled) @@ -1302,6 +1311,16 @@ def _(): object_classification_nireq.render() object_classification_reclassify_interval.render() + # Compositor Device + @gr.render(triggers=[run_tab.select]) + def _(): + show_hide_component( + compositor_device, + current_pipeline[1]["parameters"]["inference"][ + "compositor_device" + ], + ) + # Footer gr.HTML( "
" diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py index 5135b993e..e2a20cc53 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py @@ -35,9 +35,20 @@ def select_elements( parameters: dict, elements: list, ) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: + compositor_gpu_id, compositor_vaapi_suffix = self._select_gpu(parameters.get("compositor_device", "")) + detection_gpu_id, detection_vaapi_suffix = self._select_gpu(parameters.get("object_detection_device", "")) + + compositor_element = self._select_element(self.instructions.compositor, elements, compositor_gpu_id, compositor_vaapi_suffix) + encoder_element = self._select_element(self.instructions.encoder, elements, detection_gpu_id, detection_vaapi_suffix) + decoder_element = self._select_element(self.instructions.decoder, elements, detection_gpu_id, detection_vaapi_suffix) + postprocessing_element = self._select_element(self.instructions.postprocessing, elements, detection_gpu_id, detection_vaapi_suffix) + + return compositor_element, encoder_element, decoder_element, postprocessing_element + + @staticmethod + def _select_gpu(device: str) -> Tuple[int, Optional[str]]: gpu_id = -1 vaapi_suffix = None - device = parameters.get("object_detection_device", "") # Determine gpu_id and vaapi_suffix # If there is only one GPU, device name is just GPU @@ -57,37 +68,33 @@ def select_elements( else: gpu_id = -1 - def _select_element(field_dict: Dict[str, List[Tuple[str, str]]]) -> Optional[str]: - key = OTHER - if gpu_id == 0: - key = GPU_0 - elif gpu_id > 0: - key = GPU_N + return gpu_id, vaapi_suffix - pairs = field_dict.get(key, []) - # Add OTHER pairs as fallback if key is not OTHER - if key != OTHER: - pairs = pairs + field_dict.get(OTHER, []) + @staticmethod + def _select_element(field_dict: Dict[str, List[Tuple[str, str]]], elements: list, gpu_id: int, vaapi_suffix: Optional[str]) -> Optional[str]: + key = OTHER + if gpu_id == 0: + key = GPU_0 + elif gpu_id > 0: + key = GPU_N - if not pairs: - return None + pairs = field_dict.get(key, []) + # Add OTHER pairs as fallback if key is not OTHER + if key != OTHER: + pairs = pairs + field_dict.get(OTHER, []) - for search, result in pairs: - if search == "": # to support optional parameters - return result - - if VAAPI_SUFFIX_PLACEHOLDER in search or VAAPI_SUFFIX_PLACEHOLDER in result: - suffix = vaapi_suffix if vaapi_suffix is not None else "" - search = search.replace(VAAPI_SUFFIX_PLACEHOLDER, suffix) - result = result.replace(VAAPI_SUFFIX_PLACEHOLDER, suffix) - for element in elements: - if element[1] == search: - return result + if not pairs: return None - compositor_element = _select_element(self.instructions.compositor) - encoder_element = _select_element(self.instructions.encoder) - decoder_element = _select_element(self.instructions.decoder) - postprocessing_element = _select_element(self.instructions.postprocessing) + for search, result in pairs: + if search == "": # to support optional parameters + return result - return compositor_element, encoder_element, decoder_element, postprocessing_element + if VAAPI_SUFFIX_PLACEHOLDER in search or VAAPI_SUFFIX_PLACEHOLDER in result: + suffix = vaapi_suffix if vaapi_suffix is not None else "" + search = search.replace(VAAPI_SUFFIX_PLACEHOLDER, suffix) + result = result.replace(VAAPI_SUFFIX_PLACEHOLDER, suffix) + for element in elements: + if element[1] == search: + return result + return None \ No newline at end of file diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/config.yaml b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/config.yaml index 572462b13..f863d85f9 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/config.yaml +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/config.yaml @@ -34,3 +34,4 @@ parameters: - "PaddleOCR (FP32)" - "Vehicle Attributes Recognition Barrier 0039 (FP16)" classification_model_default: "PaddleOCR (FP32)" + compositor_device: false diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/config.yaml b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/config.yaml index f40eba360..282ca993d 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/config.yaml +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/config.yaml @@ -30,3 +30,4 @@ parameters: - "MobileNet V2 PyTorch (FP16)" - "ResNet-50 TF (INT8)" classification_model_default: "ResNet-50 TF (INT8)" + compositor_device: true diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/utils.py b/tools/visual-pipeline-and-platform-evaluation-tool/utils.py index a5cac4f30..6d7bdaecd 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/utils.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/utils.py @@ -62,6 +62,8 @@ def prepare_video_and_constants( "object_classification_reclassify_interval", 0.0 ) object_classification_nireq = kwargs.get("object_classification_nireq", 1) + compositor_device = str(kwargs.get("compositor_device", "")) + tracking_type = kwargs.get("tracking_type", "short-term-imageless") pipeline_watermark_enabled = kwargs.get("pipeline_watermark_enabled", True) pipeline_video_enabled = kwargs.get("pipeline_video_enabled", True) @@ -95,6 +97,7 @@ def prepare_video_and_constants( object_classification_reclassify_interval ], "object_classification_nireq": [object_classification_nireq], + "compositor_device": compositor_device.split(", "), "tracking_type": [tracking_type], "pipeline_watermark_enabled": [pipeline_watermark_enabled], "pipeline_video_enabled": [pipeline_video_enabled], From d5a6d85df8a2036672d3c2ede959bbdc93ecf3ff Mon Sep 17 00:00:00 2001 From: "Zak, Pawel" Date: Thu, 18 Sep 2025 11:49:34 +0200 Subject: [PATCH 10/13] fix linter, formatting and tests --- .../pipelines/_common/common.py | 58 +++++++++++++++---- .../pipelines/simplevs/pipeline.py | 27 +++++++-- .../pipelines/smartnvr/pipeline.py | 12 +++- .../simplevs_tests/pipeline_test.py | 19 +++--- .../smartnvr_tests/pipeline_test.py | 2 + 5 files changed, 90 insertions(+), 28 deletions(-) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py index e2a20cc53..c9890081d 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py @@ -35,15 +35,44 @@ def select_elements( parameters: dict, elements: list, ) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: - compositor_gpu_id, compositor_vaapi_suffix = self._select_gpu(parameters.get("compositor_device", "")) - detection_gpu_id, detection_vaapi_suffix = self._select_gpu(parameters.get("object_detection_device", "")) - - compositor_element = self._select_element(self.instructions.compositor, elements, compositor_gpu_id, compositor_vaapi_suffix) - encoder_element = self._select_element(self.instructions.encoder, elements, detection_gpu_id, detection_vaapi_suffix) - decoder_element = self._select_element(self.instructions.decoder, elements, detection_gpu_id, detection_vaapi_suffix) - postprocessing_element = self._select_element(self.instructions.postprocessing, elements, detection_gpu_id, detection_vaapi_suffix) - - return compositor_element, encoder_element, decoder_element, postprocessing_element + compositor_gpu_id, compositor_vaapi_suffix = self._select_gpu( + parameters.get("compositor_device", "") + ) + detection_gpu_id, detection_vaapi_suffix = self._select_gpu( + parameters.get("object_detection_device", "") + ) + + compositor_element = self._select_element( + self.instructions.compositor, + elements, + compositor_gpu_id, + compositor_vaapi_suffix, + ) + encoder_element = self._select_element( + self.instructions.encoder, + elements, + detection_gpu_id, + detection_vaapi_suffix, + ) + decoder_element = self._select_element( + self.instructions.decoder, + elements, + detection_gpu_id, + detection_vaapi_suffix, + ) + postprocessing_element = self._select_element( + self.instructions.postprocessing, + elements, + detection_gpu_id, + detection_vaapi_suffix, + ) + + return ( + compositor_element, + encoder_element, + decoder_element, + postprocessing_element, + ) @staticmethod def _select_gpu(device: str) -> Tuple[int, Optional[str]]: @@ -71,7 +100,12 @@ def _select_gpu(device: str) -> Tuple[int, Optional[str]]: return gpu_id, vaapi_suffix @staticmethod - def _select_element(field_dict: Dict[str, List[Tuple[str, str]]], elements: list, gpu_id: int, vaapi_suffix: Optional[str]) -> Optional[str]: + def _select_element( + field_dict: Dict[str, List[Tuple[str, str]]], + elements: list, + gpu_id: int, + vaapi_suffix: Optional[str], + ) -> Optional[str]: key = OTHER if gpu_id == 0: key = GPU_0 @@ -87,7 +121,7 @@ def _select_element(field_dict: Dict[str, List[Tuple[str, str]]], elements: list return None for search, result in pairs: - if search == "": # to support optional parameters + if search == "": # to support optional parameters return result if VAAPI_SUFFIX_PLACEHOLDER in search or VAAPI_SUFFIX_PLACEHOLDER in result: @@ -97,4 +131,4 @@ def _select_element(field_dict: Dict[str, List[Tuple[str, str]]], elements: list for element in elements: if element[1] == search: return result - return None \ No newline at end of file + return None diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py index b901afabb..9a077332c 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py @@ -4,11 +4,24 @@ import struct from gstpipeline import GstPipeline -from pipelines._common.common import PipelineElementsSelector, PipelineElementSelectionInstructions, VAAPI_SUFFIX_PLACEHOLDER, GPU_0, GPU_N,OTHER -from utils import get_video_resolution, UINT8_DTYPE_SIZE, VIDEO_STREAM_META_PATH, is_yolov10_model +from pipelines._common.common import ( + PipelineElementsSelector, + PipelineElementSelectionInstructions, + VAAPI_SUFFIX_PLACEHOLDER, + GPU_0, + GPU_N, + OTHER, +) +from utils import ( + get_video_resolution, + UINT8_DTYPE_SIZE, + VIDEO_STREAM_META_PATH, + is_yolov10_model, +) logger = logging.getLogger("simplevs") + class SimpleVideoStructurizationPipeline(GstPipeline): def __init__(self): super().__init__() @@ -42,7 +55,7 @@ def __init__(self): "h264parse ! " # Decoder "{decoder} ! " - "{postprocessing}" # postprocessing is optional, if present it will have a trailing " ! " + "{postprocessing}" # postprocessing is optional, if present it will have a trailing " ! " # Detection "gvafpscounter starting-frame=500 ! " "gvadetect " @@ -139,7 +152,7 @@ def __init__(self): ), ], OTHER: [ - ("", ""), # force empty string if no postprocessing is needed + ("", ""), # force empty string if no postprocessing is needed ], }, ) @@ -187,10 +200,12 @@ def evaluate( ] ): logger.error("Could not find all necessary elements for the pipeline.") - logger.error(f"Encoder: {_encoder_element}, Decoder: {_decoder_element}, Postprocessing: {_postprocessing_element}") + logger.error( + f"Encoder: {_encoder_element}, Decoder: {_decoder_element}, Postprocessing: {_postprocessing_element}" + ) return "" else: - logger.info( + logger.debug( f"Using pipeline elements - Encoder: {_encoder_element}, Decoder: {_decoder_element}, Postprocessing: {_postprocessing_element}" ) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py index df59139d1..3416b5f7f 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py @@ -5,11 +5,19 @@ import struct from gstpipeline import GstPipeline -from pipelines._common.common import PipelineElementsSelector, PipelineElementSelectionInstructions, VAAPI_SUFFIX_PLACEHOLDER, GPU_0, GPU_N,OTHER +from pipelines._common.common import ( + PipelineElementsSelector, + PipelineElementSelectionInstructions, + VAAPI_SUFFIX_PLACEHOLDER, + GPU_0, + GPU_N, + OTHER, +) from utils import UINT8_DTYPE_SIZE, VIDEO_STREAM_META_PATH, is_yolov10_model logger = logging.getLogger("smartnvr") + class SmartNVRPipeline(GstPipeline): def __init__(self): super().__init__() @@ -245,7 +253,7 @@ def evaluate( ) return "" else: - logger.info( + logger.debug( f"Using pipeline elements - Compositor: {_compositor_element}, Encoder: {_encoder_element}, Decoder: {_decoder_element}, Postprocessing: {_postprocessing_element}" ) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py index 39cb4d33c..989f20d73 100755 --- a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py @@ -110,9 +110,8 @@ def test_evaluate_gpu(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "vacompositor", "..."), ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), + ("va", "vaapidecodebin", "..."), ("va", "vapostproc", "..."), ], ) @@ -125,7 +124,9 @@ def test_evaluate_gpu(self): self.assertIn("model-proc=classification_model_proc.json", result) # Check that the decoder element is correctly used - self.assertIn("vah264dec", result) + self.assertIn( + "decodebin ! vapostproc ! video/x-raw\\(memory:VAMemory\\)", result + ) # Check that va-surface-sharing is used for pre-processing self.assertIn("pre-process-backend=va-surface-sharing", result) @@ -161,9 +162,8 @@ def test_evaluate_no_model_proc(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "vacompositor", "..."), ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), + ("va", "vaapidecodebin", "..."), ("va", "vapostproc", "..."), ], ) @@ -198,8 +198,9 @@ def test_evaluate_no_overlay(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "vah264dec", "..."), ("va", "vah264enc", "..."), + ("va", "vaapidecodebin", "..."), + ("va", "vapostproc", "..."), ], ) @@ -234,7 +235,8 @@ def test_evaluate_no_overlay_no_video(self): inference_channels=self.inference_channels, elements=[ ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), + ("va", "vaapidecodebin", "..."), + ("va", "vapostproc", "..."), ], ) @@ -269,7 +271,8 @@ def test_evaluate_no_overlay_when_video_disabled(self): inference_channels=self.inference_channels, elements=[ ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), + ("va", "vaapidecodebin", "..."), + ("va", "vapostproc", "..."), ], ) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py index 4e7a8784e..dcad3abab 100755 --- a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py @@ -215,6 +215,7 @@ def test_evaluate_gpu_0(self): "object_classification_inference_interval": 1, "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, + "compositor_device": "GPU.0", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": True, "live_preview_enabled": False, @@ -258,6 +259,7 @@ def test_evaluate_gpu_1(self): "object_classification_inference_interval": 1, "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, + "compositor_device": "GPU.1", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": True, "live_preview_enabled": False, From 5443aa4442d9a95ac5506253dc0689f5689572ed Mon Sep 17 00:00:00 2001 From: "Zak, Pawel" Date: Thu, 18 Sep 2025 15:34:31 +0200 Subject: [PATCH 11/13] move common.py up --- .../pipelines/{_common => }/common.py | 0 .../pipelines/simplevs/pipeline.py | 2 +- .../pipelines/smartnvr/pipeline.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename tools/visual-pipeline-and-platform-evaluation-tool/pipelines/{_common => }/common.py (100%) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/common.py similarity index 100% rename from tools/visual-pipeline-and-platform-evaluation-tool/pipelines/_common/common.py rename to tools/visual-pipeline-and-platform-evaluation-tool/pipelines/common.py diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py index 9a077332c..db100416c 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py @@ -4,7 +4,7 @@ import struct from gstpipeline import GstPipeline -from pipelines._common.common import ( +from pipelines.common import ( PipelineElementsSelector, PipelineElementSelectionInstructions, VAAPI_SUFFIX_PLACEHOLDER, diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py index 3416b5f7f..c77cc0e38 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py @@ -5,7 +5,7 @@ import struct from gstpipeline import GstPipeline -from pipelines._common.common import ( +from pipelines.common import ( PipelineElementsSelector, PipelineElementSelectionInstructions, VAAPI_SUFFIX_PLACEHOLDER, From 59410682a0017712ddd96ae7b41a2ee4dccaa331 Mon Sep 17 00:00:00 2001 From: "Zak, Pawel" Date: Fri, 19 Sep 2025 10:15:27 +0200 Subject: [PATCH 12/13] npu mustn't act as compositor_device --- .../app.py | 87 ++++++++++++------- 1 file changed, 55 insertions(+), 32 deletions(-) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/app.py b/tools/visual-pipeline-and-platform-evaluation-tool/app.py index 880778247..e75c51178 100755 --- a/tools/visual-pipeline-and-platform-evaluation-tool/app.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/app.py @@ -518,6 +518,50 @@ def update_inferencing_channels_label(): return gr.update(minimum=1, value=8, label=INFERENCING_CHANNELS_LABEL) +def select_preferred_devices(device_list): + """ + Returns: + preferred_device: str - preferred device for inference (GPU > NPU > CPU) + non_npu_preferred_device: str - preferred device for inference, but never NPU (GPU > CPU) + """ + # Select preferred device for inference + # 1. If any discrete GPU, pick the one with the smallest gpu_id + # 2. If any GPU, pick the one with the smallest gpu_id + # 3. Else pick NPU + # 4. Else pick CPU + + # Find discrete GPUs + discrete_gpus = [ + d + for d in device_list + if d.device_family == DeviceFamily.GPU and d.device_type == DeviceType.DISCRETE + ] + if discrete_gpus: + # Pick discrete GPU with smallest gpu_id + preferred_device = min(discrete_gpus, key=lambda d: d.gpu_id).device_name + non_npu_preferred_device = preferred_device + else: + # Find any GPU + gpus = [d for d in device_list if d.device_family == DeviceFamily.GPU] + if gpus: + # Pick GPU with smallest gpu_id + preferred_device = min(gpus, key=lambda d: d.gpu_id).device_name + non_npu_preferred_device = preferred_device + else: + # Find NPU + npus = [d for d in device_list if d.device_family == DeviceFamily.NPU] + if npus: + # Pick first NPU + preferred_device = npus[0].device_name + # For non_npu_preferred_device, fallback to CPU if no GPU + non_npu_preferred_device = "CPU" + else: + # Default to CPU + preferred_device = "CPU" + non_npu_preferred_device = "CPU" + return preferred_device, non_npu_preferred_device + + # Create the interface def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool"): """ @@ -636,39 +680,18 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool" # Inference accordion inference_accordion = gr.Accordion("Inference Parameters", open=True) - # Select preferred device for inference - # 1. If any discrete GPU, pick the one with the smallest gpu_id - # 2. If any GPU, pick the one with the smallest gpu_id - # 3. Else pick NPU - # 4. Else pick CPU device_list = device_discovery.list_devices() - # Find discrete GPUs - discrete_gpus = [ - d - for d in device_list - if d.device_family == DeviceFamily.GPU and d.device_type == DeviceType.DISCRETE - ] - if discrete_gpus: - # Pick discrete GPU with smallest gpu_id - preferred_device = min(discrete_gpus, key=lambda d: d.gpu_id).device_name - else: - # Find any GPU - gpus = [d for d in device_list if d.device_family == DeviceFamily.GPU] - if gpus: - # Pick GPU with smallest gpu_id - preferred_device = min(gpus, key=lambda d: d.gpu_id).device_name - else: - # Find NPU - npus = [d for d in device_list if d.device_family == DeviceFamily.NPU] - if npus: - # Pick first NPU - preferred_device = npus[0].device_name - else: - # Default to CPU - preferred_device = "CPU" - # Get available devices for inference + # Get available devices devices = [(device.full_device_name, device.device_name) for device in device_list] + non_npu_devices = [ + (device.full_device_name, device.device_name) + for device in device_list + if device.device_family != DeviceFamily.NPU + ] + + # Select preferred devices + preferred_device, non_npu_preferred_device = select_preferred_devices(device_list) # Object detection model # Mapping of these choices to actual model path in utils.py @@ -799,8 +822,8 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool" # Compositor device compositor_device = gr.Dropdown( label="Compositor Device", - choices=devices, - value=preferred_device, + choices=non_npu_devices, + value=non_npu_preferred_device, elem_id="compositor_device", ) From ae1d976df87c8dee4cf8187c921eceb5e74666df Mon Sep 17 00:00:00 2001 From: "Zak, Pawel" Date: Fri, 19 Sep 2025 12:30:12 +0200 Subject: [PATCH 13/13] decoding device --- .../app.py | 11 +++++++++++ .../pipelines/common.py | 7 +++++-- .../pipelines_tests/simplevs_tests/pipeline_test.py | 5 +++++ .../pipelines_tests/smartnvr_tests/pipeline_test.py | 2 ++ .../utils.py | 2 ++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/app.py b/tools/visual-pipeline-and-platform-evaluation-tool/app.py index e75c51178..aa8778c83 100755 --- a/tools/visual-pipeline-and-platform-evaluation-tool/app.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/app.py @@ -827,6 +827,14 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool" elem_id="compositor_device", ) + # Decoder device + decoding_device = gr.Dropdown( + label="Decoding Device", + choices=devices, + value=preferred_device, + elem_id="decoding_device", + ) + pipeline_watermark_enabled = gr.Checkbox( label="Overlay inference results on inference channels", value=True, @@ -896,6 +904,7 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool" components.add(object_classification_nireq) components.add(object_classification_reclassify_interval) components.add(compositor_device) + components.add(decoding_device) components.add(pipeline_watermark_enabled) components.add(pipeline_video_enabled) components.add(live_preview_enabled) @@ -1344,6 +1353,8 @@ def _(): ], ) + decoding_device.render() + # Footer gr.HTML( "
" diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/common.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/common.py index c9890081d..369298b64 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/common.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/common.py @@ -41,6 +41,9 @@ def select_elements( detection_gpu_id, detection_vaapi_suffix = self._select_gpu( parameters.get("object_detection_device", "") ) + decoding_gpu_id, decoding_vaapi_suffix = self._select_gpu( + parameters.get("decoding_device", "") + ) compositor_element = self._select_element( self.instructions.compositor, @@ -57,8 +60,8 @@ def select_elements( decoder_element = self._select_element( self.instructions.decoder, elements, - detection_gpu_id, - detection_vaapi_suffix, + decoding_gpu_id, + decoding_vaapi_suffix, ) postprocessing_element = self._select_element( self.instructions.postprocessing, diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py index 989f20d73..ae3307dac 100755 --- a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py @@ -102,6 +102,7 @@ def test_evaluate_gpu(self): "object_classification_inference_interval": 1, "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, + "decoding_device": "GPU", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": True, "pipeline_video_enabled": True, @@ -154,6 +155,7 @@ def test_evaluate_no_model_proc(self): "object_classification_inference_interval": 1, "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, + "decoding_device": "GPU", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": True, "pipeline_video_enabled": True, @@ -190,6 +192,7 @@ def test_evaluate_no_overlay(self): "object_classification_inference_interval": 1, "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, + "decoding_device": "GPU", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": False, "pipeline_video_enabled": True, @@ -226,6 +229,7 @@ def test_evaluate_no_overlay_no_video(self): "object_classification_inference_interval": 1, "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, + "decoding_device": "GPU", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": False, "pipeline_video_enabled": False, @@ -262,6 +266,7 @@ def test_evaluate_no_overlay_when_video_disabled(self): "object_classification_inference_interval": 1, "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, + "decoding_device": "GPU", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": True, "pipeline_video_enabled": False, diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py index dcad3abab..1335cf2de 100755 --- a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py @@ -216,6 +216,7 @@ def test_evaluate_gpu_0(self): "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, "compositor_device": "GPU.0", + "decoding_device": "GPU.0", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": True, "live_preview_enabled": False, @@ -260,6 +261,7 @@ def test_evaluate_gpu_1(self): "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, "compositor_device": "GPU.1", + "decoding_device": "GPU.1", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": True, "live_preview_enabled": False, diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/utils.py b/tools/visual-pipeline-and-platform-evaluation-tool/utils.py index 6d7bdaecd..93f4a6da8 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/utils.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/utils.py @@ -63,6 +63,7 @@ def prepare_video_and_constants( ) object_classification_nireq = kwargs.get("object_classification_nireq", 1) compositor_device = str(kwargs.get("compositor_device", "")) + decoding_device = str(kwargs.get("decoding_device", "")) tracking_type = kwargs.get("tracking_type", "short-term-imageless") pipeline_watermark_enabled = kwargs.get("pipeline_watermark_enabled", True) @@ -98,6 +99,7 @@ def prepare_video_and_constants( ], "object_classification_nireq": [object_classification_nireq], "compositor_device": compositor_device.split(", "), + "decoding_device": decoding_device.split(", "), "tracking_type": [tracking_type], "pipeline_watermark_enabled": [pipeline_watermark_enabled], "pipeline_video_enabled": [pipeline_video_enabled],