diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/app.py b/tools/visual-pipeline-and-platform-evaluation-tool/app.py index fed4dfee0..aa8778c83 100755 --- a/tools/visual-pipeline-and-platform-evaluation-tool/app.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/app.py @@ -518,6 +518,50 @@ def update_inferencing_channels_label(): return gr.update(minimum=1, value=8, label=INFERENCING_CHANNELS_LABEL) +def select_preferred_devices(device_list): + """ + Returns: + preferred_device: str - preferred device for inference (GPU > NPU > CPU) + non_npu_preferred_device: str - preferred device for inference, but never NPU (GPU > CPU) + """ + # Select preferred device for inference + # 1. If any discrete GPU, pick the one with the smallest gpu_id + # 2. If any GPU, pick the one with the smallest gpu_id + # 3. Else pick NPU + # 4. Else pick CPU + + # Find discrete GPUs + discrete_gpus = [ + d + for d in device_list + if d.device_family == DeviceFamily.GPU and d.device_type == DeviceType.DISCRETE + ] + if discrete_gpus: + # Pick discrete GPU with smallest gpu_id + preferred_device = min(discrete_gpus, key=lambda d: d.gpu_id).device_name + non_npu_preferred_device = preferred_device + else: + # Find any GPU + gpus = [d for d in device_list if d.device_family == DeviceFamily.GPU] + if gpus: + # Pick GPU with smallest gpu_id + preferred_device = min(gpus, key=lambda d: d.gpu_id).device_name + non_npu_preferred_device = preferred_device + else: + # Find NPU + npus = [d for d in device_list if d.device_family == DeviceFamily.NPU] + if npus: + # Pick first NPU + preferred_device = npus[0].device_name + # For non_npu_preferred_device, fallback to CPU if no GPU + non_npu_preferred_device = "CPU" + else: + # Default to CPU + preferred_device = "CPU" + non_npu_preferred_device = "CPU" + return preferred_device, non_npu_preferred_device + + # Create the interface def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool"): """ @@ -636,39 +680,18 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool" # Inference accordion inference_accordion = gr.Accordion("Inference Parameters", open=True) - # Select preferred device for inference - # 1. If any discrete GPU, pick the one with the smallest gpu_id - # 2. If any GPU, pick the one with the smallest gpu_id - # 3. Else pick NPU - # 4. Else pick CPU device_list = device_discovery.list_devices() - # Find discrete GPUs - discrete_gpus = [ - d - for d in device_list - if d.device_family == DeviceFamily.GPU and d.device_type == DeviceType.DISCRETE - ] - if discrete_gpus: - # Pick discrete GPU with smallest gpu_id - preferred_device = min(discrete_gpus, key=lambda d: d.gpu_id).device_name - else: - # Find any GPU - gpus = [d for d in device_list if d.device_family == DeviceFamily.GPU] - if gpus: - # Pick GPU with smallest gpu_id - preferred_device = min(gpus, key=lambda d: d.gpu_id).device_name - else: - # Find NPU - npus = [d for d in device_list if d.device_family == DeviceFamily.NPU] - if npus: - # Pick first NPU - preferred_device = npus[0].device_name - else: - # Default to CPU - preferred_device = "CPU" - # Get available devices for inference + # Get available devices devices = [(device.full_device_name, device.device_name) for device in device_list] + non_npu_devices = [ + (device.full_device_name, device.device_name) + for device in device_list + if device.device_family != DeviceFamily.NPU + ] + + # Select preferred devices + preferred_device, non_npu_preferred_device = select_preferred_devices(device_list) # Object detection model # Mapping of these choices to actual model path in utils.py @@ -796,6 +819,22 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool" elem_id="object_classification_reclassify_interval", ) + # Compositor device + compositor_device = gr.Dropdown( + label="Compositor Device", + choices=non_npu_devices, + value=non_npu_preferred_device, + elem_id="compositor_device", + ) + + # Decoder device + decoding_device = gr.Dropdown( + label="Decoding Device", + choices=devices, + value=preferred_device, + elem_id="decoding_device", + ) + pipeline_watermark_enabled = gr.Checkbox( label="Overlay inference results on inference channels", value=True, @@ -864,6 +903,8 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool" components.add(object_classification_inference_interval) components.add(object_classification_nireq) components.add(object_classification_reclassify_interval) + components.add(compositor_device) + components.add(decoding_device) components.add(pipeline_watermark_enabled) components.add(pipeline_video_enabled) components.add(live_preview_enabled) @@ -1302,6 +1343,18 @@ def _(): object_classification_nireq.render() object_classification_reclassify_interval.render() + # Compositor Device + @gr.render(triggers=[run_tab.select]) + def _(): + show_hide_component( + compositor_device, + current_pipeline[1]["parameters"]["inference"][ + "compositor_device" + ], + ) + + decoding_device.render() + # Footer gr.HTML( "
" diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/common.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/common.py new file mode 100644 index 000000000..369298b64 --- /dev/null +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/common.py @@ -0,0 +1,137 @@ +from typing import List, Tuple, Dict, Optional + +# Keys for device selection +GPU_0 = "GPU_0" +GPU_N = "GPU_N" +OTHER = "OTHER" +# Placeholder for vaapi_suffix to be replaced at runtime +VAAPI_SUFFIX_PLACEHOLDER = "{vaapi_suffix}" + + +class PipelineElementSelectionInstructions: + def __init__( + self, + compositor: Optional[Dict[str, List[Tuple[str, str]]]] = None, + encoder: Optional[Dict[str, List[Tuple[str, str]]]] = None, + decoder: Optional[Dict[str, List[Tuple[str, str]]]] = None, + postprocessing: Optional[Dict[str, List[Tuple[str, str]]]] = None, + ): + # Each field is a dict: key is GPU_0, GPU_N, or OTHER, value is list of (search, result) + self.compositor = compositor or {} + self.encoder = encoder or {} + self.decoder = decoder or {} + self.postprocessing = postprocessing or {} + + +class PipelineElementsSelector: + def __init__( + self, + selection_instructions: PipelineElementSelectionInstructions, + ): + self.instructions = selection_instructions + + def select_elements( + self, + parameters: dict, + elements: list, + ) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: + compositor_gpu_id, compositor_vaapi_suffix = self._select_gpu( + parameters.get("compositor_device", "") + ) + detection_gpu_id, detection_vaapi_suffix = self._select_gpu( + parameters.get("object_detection_device", "") + ) + decoding_gpu_id, decoding_vaapi_suffix = self._select_gpu( + parameters.get("decoding_device", "") + ) + + compositor_element = self._select_element( + self.instructions.compositor, + elements, + compositor_gpu_id, + compositor_vaapi_suffix, + ) + encoder_element = self._select_element( + self.instructions.encoder, + elements, + detection_gpu_id, + detection_vaapi_suffix, + ) + decoder_element = self._select_element( + self.instructions.decoder, + elements, + decoding_gpu_id, + decoding_vaapi_suffix, + ) + postprocessing_element = self._select_element( + self.instructions.postprocessing, + elements, + detection_gpu_id, + detection_vaapi_suffix, + ) + + return ( + compositor_element, + encoder_element, + decoder_element, + postprocessing_element, + ) + + @staticmethod + def _select_gpu(device: str) -> Tuple[int, Optional[str]]: + gpu_id = -1 + vaapi_suffix = None + + # Determine gpu_id and vaapi_suffix + # If there is only one GPU, device name is just GPU + # If there is more than one GPU, device names are like GPU.0, GPU.1, ... + if device == "GPU": + gpu_id = 0 + elif device.startswith("GPU."): + try: + gpu_index = int(device.split(".")[1]) + if gpu_index == 0: + gpu_id = 0 + elif gpu_index > 0: + vaapi_suffix = str(128 + gpu_index) + gpu_id = gpu_index + except (IndexError, ValueError): + gpu_id = -1 + else: + gpu_id = -1 + + return gpu_id, vaapi_suffix + + @staticmethod + def _select_element( + field_dict: Dict[str, List[Tuple[str, str]]], + elements: list, + gpu_id: int, + vaapi_suffix: Optional[str], + ) -> Optional[str]: + key = OTHER + if gpu_id == 0: + key = GPU_0 + elif gpu_id > 0: + key = GPU_N + + pairs = field_dict.get(key, []) + # Add OTHER pairs as fallback if key is not OTHER + if key != OTHER: + pairs = pairs + field_dict.get(OTHER, []) + + if not pairs: + return None + + for search, result in pairs: + if search == "": # to support optional parameters + return result + + if VAAPI_SUFFIX_PLACEHOLDER in search or VAAPI_SUFFIX_PLACEHOLDER in result: + suffix = vaapi_suffix if vaapi_suffix is not None else "" + search = search.replace(VAAPI_SUFFIX_PLACEHOLDER, suffix) + result = result.replace(VAAPI_SUFFIX_PLACEHOLDER, suffix) + for element in elements: + if element[1] == search: + return result + return None diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/config.yaml b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/config.yaml index 572462b13..f863d85f9 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/config.yaml +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/config.yaml @@ -34,3 +34,4 @@ parameters: - "PaddleOCR (FP32)" - "Vehicle Attributes Recognition Barrier 0039 (FP16)" classification_model_default: "PaddleOCR (FP32)" + compositor_device: false diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py index c94651afc..db100416c 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/simplevs/pipeline.py @@ -4,6 +4,14 @@ import struct from gstpipeline import GstPipeline +from pipelines.common import ( + PipelineElementsSelector, + PipelineElementSelectionInstructions, + VAAPI_SUFFIX_PLACEHOLDER, + GPU_0, + GPU_N, + OTHER, +) from utils import ( get_video_resolution, UINT8_DTYPE_SIZE, @@ -11,6 +19,8 @@ is_yolov10_model, ) +logger = logging.getLogger("simplevs") + class SimpleVideoStructurizationPipeline(GstPipeline): def __init__(self): @@ -22,39 +32,58 @@ def __init__(self): (330, 110, 445, 170, "Inference", "Object Detection"), ] + # Add shmsink for live-streaming (shared memory) + self._shmsink = ( + "shmsink socket-path=/tmp/shared_memory/video_stream " + "wait-for-connection=false " + "sync=true " + "name=shmsink0 " + ) + + # shmsink branch for live preview (BGR format), width/height will be formatted in evaluate + self._shmsink_branch = ( + "tee name=livetee " + "livetee. ! queue2 ! {encoder} ! h264parse ! mp4mux ! filesink location={VIDEO_OUTPUT_PATH} async=false " + "livetee. ! queue2 ! videoconvert ! video/x-raw,format=BGR,width={output_width},height={output_height} ! {shmsink} " + ) + self._inference_stream_decode_detect_track = ( # Input - "filesrc location={VIDEO_PATH} ! " + "filesrc " + " location={VIDEO_PATH} ! " + "qtdemux ! " + "h264parse ! " # Decoder "{decoder} ! " + "{postprocessing}" # postprocessing is optional, if present it will have a trailing " ! " # Detection "gvafpscounter starting-frame=500 ! " "gvadetect " - " {detection_model_config} " - " model-instance-id=detect0 " - " device={object_detection_device} " - " pre-process-backend={object_detection_pre_process_backend} " - " batch-size={object_detection_batch_size} " - " inference-interval={object_detection_inference_interval} " - " {ie_config_parameter} " - " nireq={object_detection_nireq} ! " - "queue ! " + " {detection_model_config} " + " model-instance-id=detect0 " + " pre-process-backend={object_detection_pre_process_backend} " + " device={object_detection_device} " + " batch-size={object_detection_batch_size} " + " inference-interval={object_detection_inference_interval} " + " {ie_config_parameter} " + " nireq={object_detection_nireq} ! " + "queue2 ! " "gvatrack " " tracking-type={tracking_type} ! " - "queue ! " + "queue2 ! " ) self._inference_stream_classify = ( "gvaclassify " - " {classification_model_config} " - " model-instance-id=classify0 " - " device={object_classification_device} " - " pre-process-backend={object_classification_pre_process_backend} " - " batch-size={object_classification_batch_size} " - " inference-interval={object_classification_inference_interval} " - " nireq={object_classification_nireq} " - " reclassify-interval={object_classification_reclassify_interval} ! " - "queue ! " + " {classification_model_config} " + " model-instance-id=classify0 " + " pre-process-backend={object_classification_pre_process_backend} " + " device={object_classification_device} " + " batch-size={object_classification_batch_size} " + " inference-interval={object_classification_inference_interval} " + " nireq={object_classification_nireq} " + " reclassify-interval={object_classification_reclassify_interval} ! " + "queue2 ! " ) self._inference_stream_metadata_processing = ( @@ -68,22 +97,65 @@ def __init__(self): ) self._inference_output_stream = ( - "{encoder} ! h264parse ! mp4mux ! filesink location={VIDEO_OUTPUT_PATH} " - ) - - # Add shmsink for live preview (shared memory) - self._shmsink = ( - "shmsink socket-path=/tmp/shared_memory/video_stream " - "wait-for-connection=false " - "sync=true " - "name=shmsink0 " + "{encoder} ! h264parse ! mp4mux ! filesink location={VIDEO_OUTPUT_PATH} " ) - # shmsink branch for live preview (BGR format), width/height will be formatted in evaluate - self._shmsink_branch = ( - "tee name=livetee " - "livetee. ! queue2 ! {encoder} ! h264parse ! mp4mux ! filesink location={VIDEO_OUTPUT_PATH} async=false " - "livetee. ! queue2 ! videoconvert ! video/x-raw,format=BGR,width={width},height={height} ! {shmsink} " + self._selector = PipelineElementsSelector( + PipelineElementSelectionInstructions( + encoder={ + GPU_0: [ + ("vah264lpenc", "vah264lpenc"), + ("vah264enc", "vah264enc"), + ], + GPU_N: [ + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264lpenc", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264lpenc", + ), + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264enc", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264enc", + ), + ], + OTHER: [ + ("x264enc", "x264enc bitrate=16000 speed-preset=superfast"), + ], + }, + decoder={ + GPU_0: [ + ( + "vaapidecodebin", + "vaapidecodebin", + ), + ], + GPU_N: [ + ( + "vaapidecodebin", + "vaapidecodebin", + ), + ], + OTHER: [ + ("decodebin", "decodebin"), + ], + }, + postprocessing={ + GPU_0: [ + ( + "vapostproc", + "vapostproc ! video/x-raw\\(memory:VAMemory\\) ! ", + ), + ], + GPU_N: [ + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}postproc", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}postproc ! video/x-raw\\(memory:VAMemory\\) ! ", + ), + ], + OTHER: [ + ("", ""), # force empty string if no postprocessing is needed + ], + }, + ) ) def evaluate( @@ -97,44 +169,47 @@ def evaluate( if elements is None: elements = [] - # Set decoder element based on device - _decoder_element = ( - "decodebin3 " - if parameters["object_detection_device"] in ["CPU", "NPU"] - else "decodebin3 ! vapostproc ! video/x-raw\\(memory:VAMemory\\)" - ) - - # Set encoder element based on device - _encoder_element = next( - ("vah264enc" for element in elements if element[1] == "vah264enc"), - next( - ("vah264lpenc" for element in elements if element[1] == "vah264lpenc"), - next( - ( - "x264enc bitrate=16000 speed-preset=superfast" - for element in elements - if element[1] == "x264enc" - ), - None, # Fallback to None if no encoder is found - ), - ), - ) - - # Set pre process backed for object detection + # Set pre-process backend for object detection parameters["object_detection_pre_process_backend"] = ( "opencv" if parameters["object_detection_device"] in ["CPU", "NPU"] else "va-surface-sharing" ) - # Set pre process backed for object classification + # Set pre-process backend for object classification parameters["object_classification_pre_process_backend"] = ( "opencv" if parameters["object_classification_device"] in ["CPU", "NPU"] else "va-surface-sharing" ) - # Set model config for object detection + # Use PipelineElementsSelector for element selection + ( + _, + _encoder_element, + _decoder_element, + _postprocessing_element, + ) = self._selector.select_elements(parameters, elements) + + # If any of the essential elements is not found, log an error and return an empty string + # _postprocessing_element is optional, so don't include it in the check + if not all( + [ + _encoder_element, + _decoder_element, + ] + ): + logger.error("Could not find all necessary elements for the pipeline.") + logger.error( + f"Encoder: {_encoder_element}, Decoder: {_decoder_element}, Postprocessing: {_postprocessing_element}" + ) + return "" + else: + logger.debug( + f"Using pipeline elements - Encoder: {_encoder_element}, Decoder: {_decoder_element}, Postprocessing: {_postprocessing_element}" + ) + + # Handle object detection parameters and constants detection_model_config = ( f"model={constants['OBJECT_DETECTION_MODEL_PATH']} " f"model-proc={constants['OBJECT_DETECTION_MODEL_PROC']} " @@ -147,34 +222,42 @@ def evaluate( # Set inference config parameter for GPU if using YOLOv10 ie_config_parameter = "" - if parameters["object_detection_device"] == "GPU" and is_yolov10_model( - constants["OBJECT_DETECTION_MODEL_PATH"] - ): + if parameters.get("object_detection_device", "").startswith( + "GPU" + ) and is_yolov10_model(constants["OBJECT_DETECTION_MODEL_PATH"]): ie_config_parameter = "ie-config=GPU_DISABLE_WINOGRAD_CONVOLUTION=YES" - streams = "" - # Prepare shmsink and meta if live_preview_enabled - width = 0 - height = 0 + output_width = 0 + output_height = 0 if parameters["live_preview_enabled"]: # Get resolution using get_video_resolution video_path = constants.get("VIDEO_PATH", "") - width, height = get_video_resolution(video_path) + output_width, output_height = get_video_resolution(video_path) + # Write meta file for live preview try: os.makedirs("/tmp/shared_memory", exist_ok=True) with open(VIDEO_STREAM_META_PATH, "wb") as f: - # height, width, dtype_size=UINT8_DTYPE_SIZE (uint8) - f.write(struct.pack("III", height, width, UINT8_DTYPE_SIZE)) + # height=output_height, width=output_width, dtype_size=UINT8_DTYPE_SIZE (uint8) + f.write( + struct.pack( + "III", output_height, output_width, UINT8_DTYPE_SIZE + ) + ) except Exception as e: - logging.warning(f"Could not write shared memory meta file: {e}") + logger.warning(f"Could not write shared memory meta file: {e}") + + # Create the streams + streams = "" + # Handle inference channels for i in range(inference_channels): streams += self._inference_stream_decode_detect_track.format( **parameters, **constants, decoder=_decoder_element, + postprocessing=_postprocessing_element, detection_model_config=detection_model_config, ie_config_parameter=ie_config_parameter, ) @@ -225,8 +308,8 @@ def evaluate( streams += self._shmsink_branch.format( **constants, encoder=_encoder_element, - width=width, - height=height, + output_width=output_width, + output_height=output_height, shmsink=self._shmsink, ) else: @@ -236,4 +319,5 @@ def evaluate( else: streams += "fakesink " + # Evaluate the pipeline return "gst-launch-1.0 -q " + streams diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/config.yaml b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/config.yaml index f40eba360..282ca993d 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/config.yaml +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/config.yaml @@ -30,3 +30,4 @@ parameters: - "MobileNet V2 PyTorch (FP16)" - "ResNet-50 TF (INT8)" classification_model_default: "ResNet-50 TF (INT8)" + compositor_device: true diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py index bc613956f..c77cc0e38 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/pipeline.py @@ -5,8 +5,18 @@ import struct from gstpipeline import GstPipeline +from pipelines.common import ( + PipelineElementsSelector, + PipelineElementSelectionInstructions, + VAAPI_SUFFIX_PLACEHOLDER, + GPU_0, + GPU_N, + OTHER, +) from utils import UINT8_DTYPE_SIZE, VIDEO_STREAM_META_PATH, is_yolov10_model +logger = logging.getLogger("smartnvr") + class SmartNVRPipeline(GstPipeline): def __init__(self): @@ -20,7 +30,7 @@ def __init__(self): self._sink = "sink_{id}::xpos={xpos} sink_{id}::ypos={ypos} sink_{id}::alpha=1 " - # Add shmsink for live streaming (shared memory) + # Add shmsink for live-streaming (shared memory) self._shmsink = ( "shmsink socket-path=/tmp/shared_memory/video_stream " "wait-for-connection=false " @@ -65,6 +75,7 @@ def __init__(self): ) self._inference_stream_decode_detect_track = ( + # Input "filesrc " " location={VIDEO_PATH} ! " "qtdemux ! " @@ -76,7 +87,9 @@ def __init__(self): " location=/tmp/stream{id}.mp4 " "t{id}. ! " "queue2 ! " + # Decoder "{decoder} ! " + # Detection "gvafpscounter starting-frame=500 ! " "gvadetect " " {detection_model_config} " @@ -126,6 +139,72 @@ def __init__(self): "comp.sink_{id} " ) + self._selector = PipelineElementsSelector( + PipelineElementSelectionInstructions( + compositor={ + GPU_0: [ + ("vacompositor", "vacompositor"), + ], + GPU_N: [ + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}compositor", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}compositor", + ), + ], + OTHER: [ + ("compositor", "compositor"), + ], + }, + encoder={ + GPU_0: [ + ("vah264lpenc", "vah264lpenc"), + ("vah264enc", "vah264enc"), + ], + GPU_N: [ + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264lpenc", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264lpenc", + ), + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264enc", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264enc", + ), + ], + OTHER: [ + ("x264enc", "x264enc bitrate=16000 speed-preset=superfast"), + ], + }, + decoder={ + GPU_0: [ + ("vah264dec", "vah264dec ! video/x-raw\\(memory:VAMemory\\)"), + ], + GPU_N: [ + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264dec", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}h264dec ! video/x-raw\\(memory:VAMemory\\)", + ), + ], + OTHER: [ + ("decodebin", "decodebin"), + ], + }, + postprocessing={ + GPU_0: [ + ("vapostproc", "vapostproc"), + ], + GPU_N: [ + ( + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}postproc", + f"varenderD{VAAPI_SUFFIX_PLACEHOLDER}postproc", + ), + ], + OTHER: [ + ("videoscale", "videoscale"), + ], + }, + ) + ) + def evaluate( self, constants: dict, @@ -137,20 +216,65 @@ def evaluate( if elements is None: elements = [] - # Set pre process backed for object detection + # Set pre-process backend for object detection parameters["object_detection_pre_process_backend"] = ( "opencv" if parameters["object_detection_device"] in ["CPU", "NPU"] else "va-surface-sharing" ) - # Set pre process backed for object classification + # Set pre-process backend for object classification parameters["object_classification_pre_process_backend"] = ( "opencv" if parameters["object_classification_device"] in ["CPU", "NPU"] else "va-surface-sharing" ) + # Use PipelineElementsSelector for element selection + ( + _compositor_element, + _encoder_element, + _decoder_element, + _postprocessing_element, + ) = self._selector.select_elements(parameters, elements) + + # If any of the essential elements is not found, log an error and return an empty string + if not all( + [ + _compositor_element, + _encoder_element, + _decoder_element, + _postprocessing_element, + ] + ): + logger.error("Could not find all necessary elements for the pipeline.") + logger.error( + f"Compositor: {_compositor_element}, Encoder: {_encoder_element}, Decoder: {_decoder_element}, Postprocessing: {_postprocessing_element}" + ) + return "" + else: + logger.debug( + f"Using pipeline elements - Compositor: {_compositor_element}, Encoder: {_encoder_element}, Decoder: {_decoder_element}, Postprocessing: {_postprocessing_element}" + ) + + # Handle object detection parameters and constants + detection_model_config = ( + f"model={constants['OBJECT_DETECTION_MODEL_PATH']} " + f"model-proc={constants['OBJECT_DETECTION_MODEL_PROC']} " + ) + + if not constants["OBJECT_DETECTION_MODEL_PROC"]: + detection_model_config = ( + f"model={constants['OBJECT_DETECTION_MODEL_PATH']} " + ) + + # Set inference config parameter for GPU if using YOLOv10 + ie_config_parameter = "" + if parameters.get("object_detection_device", "").startswith( + "GPU" + ) and is_yolov10_model(constants["OBJECT_DETECTION_MODEL_PATH"]): + ie_config_parameter = "ie-config=GPU_DISABLE_WINOGRAD_CONVOLUTION=YES" + # Compute total number of channels channels = regular_channels + inference_channels @@ -162,125 +286,34 @@ def evaluate( ypos = 360 * (i // grid_size) sinks += self._sink.format(id=i, xpos=xpos, ypos=ypos) - # Find the available compositor in elements dynamically - if ( - parameters["object_detection_device"].startswith("GPU.") - and int(parameters["object_detection_device"].split(".")[1]) > 0 - ): - gpu_index = parameters["object_detection_device"].split(".")[1] - # Map GPU index to the corresponding VAAPI element suffix (e.g., "129" for GPU.1) - vaapi_suffix = str( - 128 + int(gpu_index) - ) # 128 + 1 = 129, 128 + 2 = 130, etc. - _compositor_element = f"varenderD{vaapi_suffix}compositor" - else: - _compositor_element = next( - ( - "vacompositor" - for element in elements - if element[1] == "vacompositor" - ), - next( - ( - "compositor" - for element in elements - if element[1] == "compositor" - ), - None, # Fallback to None if no compositor is found - ), - ) - - # Find the available encoder dynamically - if ( - parameters["object_detection_device"].startswith("GPU.") - and int(parameters["object_detection_device"].split(".")[1]) > 0 - ): - gpu_index = parameters["object_detection_device"].split(".")[1] - # Map GPU index to the corresponding VAAPI element suffix (e.g., "129" for GPU.1) - vaapi_suffix = str( - 128 + int(gpu_index) - ) # 128 + 1 = 129, 128 + 2 = 130, etc. - _encoder_element = f"varenderD{vaapi_suffix}h264lpenc" - else: - # Fallback to default encoder if no specific GPU is selected - _encoder_element = next( - ("vah264lpenc" for element in elements if element[1] == "vah264lpenc"), - next( - ("vah264enc" for element in elements if element[1] == "vah264enc"), - next( - ( - "x264enc bitrate=16000 speed-preset=superfast" - for element in elements - if element[1] == "x264enc" - ), - None, # Fallback to None if no encoder is found - ), - ), - ) + # Prepare shmsink and meta if live_preview_enabled + output_width = 0 + output_height = 0 + if parameters["live_preview_enabled"]: + # Calculate output video size for grid layout to ensure same resolution for shmsink and output file + output_width = 640 * grid_size + output_height = 360 * ( + (channels + grid_size - 1) // grid_size + ) # ceil(channels / grid_size) - # Find the available decoder and postprocessing elements dynamically - if ( - parameters["object_detection_device"].startswith("GPU.") - and int(parameters["object_detection_device"].split(".")[1]) > 0 - ): - # Extract the GPU index (e.g., "1" from "GPU.1") - gpu_index = parameters["object_detection_device"].split(".")[1] - # Map GPU index to the corresponding VAAPI element suffix (e.g., "129" for GPU.1) - vaapi_suffix = str( - 128 + int(gpu_index) - ) # 128 + 1 = 129, 128 + 2 = 130, etc. - _decoder_element = ( - f"varenderD{vaapi_suffix}h264dec ! video/x-raw(memory:VAMemory)" - ) - _postprocessing_element = f"varenderD{vaapi_suffix}postproc" - else: - # Fallback to default elements if no specific GPU is selected - _decoder_element = next( - ( - "vah264dec ! video/x-raw(memory:VAMemory)" - for element in elements - if element[1] == "vah264dec" - ), - next( - ("decodebin" for element in elements if element[1] == "decodebin"), - None, # Fallback to None if no decoder is found - ), - ) - _postprocessing_element = next( - ("vapostproc" for element in elements if element[1] == "vapostproc"), - next( - ( - "videoscale" - for element in elements - if element[1] == "videoscale" - ), - None, # Fallback to None if no postprocessing is found - ), - ) + # Write meta file for live preview + try: + os.makedirs("/tmp/shared_memory", exist_ok=True) + with open(VIDEO_STREAM_META_PATH, "wb") as f: + # height=output_height, width=output_width, dtype_size=UINT8_DTYPE_SIZE (uint8) + f.write( + struct.pack( + "III", output_height, output_width, UINT8_DTYPE_SIZE + ) + ) + except Exception as e: + logger.warning(f"Could not write shared memory meta file: {e}") # Create the streams streams = "" # Handle inference channels for i in range(inference_channels): - # Handle object detection parameters and constants - detection_model_config = ( - f"model={constants['OBJECT_DETECTION_MODEL_PATH']} " - f"model-proc={constants['OBJECT_DETECTION_MODEL_PROC']} " - ) - - if not constants["OBJECT_DETECTION_MODEL_PROC"]: - detection_model_config = ( - f"model={constants['OBJECT_DETECTION_MODEL_PATH']} " - ) - - # Set inference config parameter for GPU if using YOLOv10 - ie_config_parameter = "" - if parameters["object_detection_device"] == "GPU" and is_yolov10_model( - constants["OBJECT_DETECTION_MODEL_PATH"] - ): - ie_config_parameter = "ie-config=GPU_DISABLE_WINOGRAD_CONVOLUTION=YES" - streams += self._inference_stream_decode_detect_track.format( **parameters, **constants, @@ -296,6 +329,7 @@ def evaluate( constants["OBJECT_CLASSIFICATION_MODEL_PATH"] == "Disabled" or parameters["object_classification_device"] == "Disabled" ): + # Set model config for object classification classification_model_config = ( f"model={constants['OBJECT_CLASSIFICATION_MODEL_PATH']} " f"model-proc={constants['OBJECT_CLASSIFICATION_MODEL_PROC']} " @@ -313,10 +347,11 @@ def evaluate( classification_model_config=classification_model_config, ) - # Overlay inference results on the inferenced video if enabled + # Overlay inference results on the inferred video if enabled if parameters["pipeline_watermark_enabled"]: streams += "gvawatermark ! " + # Metadata processing and publishing streams += self._inference_stream_metadata_processing.format( **parameters, **constants, @@ -338,7 +373,6 @@ def evaluate( **constants, id=i, decoder=_decoder_element, - postprocessing=_postprocessing_element, ) # sink to compositor or fake sink depending on the compose flag streams += self._sink_to_compositor.format( @@ -350,25 +384,7 @@ def evaluate( ) # Compose pipeline depending on live_preview_enabled if parameters["live_preview_enabled"]: - # Calculate output video size for grid layout to ensure same resolution for shmsink and output file - output_width = 640 * grid_size - output_height = 360 * ( - (channels + grid_size - 1) // grid_size - ) # ceil(channels / grid_size) - # Always produce both file and live stream outputs - try: - os.makedirs("/tmp/shared_memory", exist_ok=True) - with open(VIDEO_STREAM_META_PATH, "wb") as f: - # width=output_height, height=output_width, dtype_size=UINT8_DTYPE_SIZE (uint8) - f.write( - struct.pack( - "III", output_height, output_width, UINT8_DTYPE_SIZE - ) - ) - except Exception as e: - logging.warning(f"Could not write shared memory meta file: {e}") - streams = ( self._compositor_with_tee.format( **constants, diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py index 5fe7b181f..ae3307dac 100755 --- a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/simplevs_tests/pipeline_test.py @@ -62,10 +62,10 @@ def test_evaluate_cpu(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "decodebin3", "..."), - ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), - ("va", "vapostproc", "..."), + ("va", "compositor", "..."), + ("va", "x264enc", "..."), + ("va", "decodebin", "..."), + ("va", "videoscale", "..."), ], ) @@ -77,7 +77,7 @@ def test_evaluate_cpu(self): self.assertIn("model-proc=classification_model_proc.json", result) # Check that the decoder element is correctly used - self.assertIn("decodebin3", result) + self.assertIn("decodebin", result) # Check that opencv is used for pre-processing self.assertIn("pre-process-backend=opencv", result) @@ -102,6 +102,7 @@ def test_evaluate_gpu(self): "object_classification_inference_interval": 1, "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, + "decoding_device": "GPU", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": True, "pipeline_video_enabled": True, @@ -110,9 +111,8 @@ def test_evaluate_gpu(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "decodebin3", "..."), ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), + ("va", "vaapidecodebin", "..."), ("va", "vapostproc", "..."), ], ) @@ -126,7 +126,7 @@ def test_evaluate_gpu(self): # Check that the decoder element is correctly used self.assertIn( - "decodebin3 ! vapostproc ! video/x-raw\\(memory:VAMemory\\)", result + "decodebin ! vapostproc ! video/x-raw\\(memory:VAMemory\\)", result ) # Check that va-surface-sharing is used for pre-processing @@ -155,6 +155,7 @@ def test_evaluate_no_model_proc(self): "object_classification_inference_interval": 1, "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, + "decoding_device": "GPU", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": True, "pipeline_video_enabled": True, @@ -163,9 +164,8 @@ def test_evaluate_no_model_proc(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "decodebin3", "..."), ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), + ("va", "vaapidecodebin", "..."), ("va", "vapostproc", "..."), ], ) @@ -192,6 +192,7 @@ def test_evaluate_no_overlay(self): "object_classification_inference_interval": 1, "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, + "decoding_device": "GPU", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": False, "pipeline_video_enabled": True, @@ -200,8 +201,9 @@ def test_evaluate_no_overlay(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "decodebin3", "..."), ("va", "vah264enc", "..."), + ("va", "vaapidecodebin", "..."), + ("va", "vapostproc", "..."), ], ) @@ -227,6 +229,7 @@ def test_evaluate_no_overlay_no_video(self): "object_classification_inference_interval": 1, "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, + "decoding_device": "GPU", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": False, "pipeline_video_enabled": False, @@ -235,8 +238,9 @@ def test_evaluate_no_overlay_no_video(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "decodebin3", "..."), ("va", "vah264enc", "..."), + ("va", "vaapidecodebin", "..."), + ("va", "vapostproc", "..."), ], ) @@ -262,6 +266,7 @@ def test_evaluate_no_overlay_when_video_disabled(self): "object_classification_inference_interval": 1, "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, + "decoding_device": "GPU", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": True, "pipeline_video_enabled": False, @@ -270,8 +275,9 @@ def test_evaluate_no_overlay_when_video_disabled(self): regular_channels=0, inference_channels=self.inference_channels, elements=[ - ("va", "decodebin3", "..."), ("va", "vah264enc", "..."), + ("va", "vaapidecodebin", "..."), + ("va", "vapostproc", "..."), ], ) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py index d4ee7d71b..1335cf2de 100755 --- a/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/tests/pipelines_tests/smartnvr_tests/pipeline_test.py @@ -73,10 +73,10 @@ def test_evaluate_no_overlay(self): regular_channels=self.regular_channels, inference_channels=self.inference_channels, elements=[ - ("va", "vacompositor", "..."), - ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), - ("va", "vapostproc", "..."), + ("va", "compositor", "..."), + ("va", "x264enc", "..."), + ("va", "decodebin", "..."), + ("va", "videoscale", "..."), ], ) @@ -109,10 +109,10 @@ def test_evaluate_classification_disabled_by_device(self): regular_channels=self.regular_channels, inference_channels=self.inference_channels, elements=[ - ("va", "vacompositor", "..."), - ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), - ("va", "vapostproc", "..."), + ("va", "compositor", "..."), + ("va", "x264enc", "..."), + ("va", "decodebin", "..."), + ("va", "videoscale", "..."), ], ) @@ -149,10 +149,10 @@ def test_evaluate_classification_disabled_by_model(self): regular_channels=self.regular_channels, inference_channels=self.inference_channels, elements=[ - ("va", "vacompositor", "..."), - ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), - ("va", "vapostproc", "..."), + ("va", "compositor", "..."), + ("va", "x264enc", "..."), + ("va", "decodebin", "..."), + ("va", "videoscale", "..."), ], ) @@ -182,10 +182,10 @@ def test_evaluate_cpu(self): regular_channels=self.regular_channels, inference_channels=self.inference_channels, elements=[ - ("va", "vacompositor", "..."), - ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), - ("va", "vapostproc", "..."), + ("va", "compositor", "..."), + ("va", "x264enc", "..."), + ("va", "decodebin", "..."), + ("va", "videoscale", "..."), ], ) @@ -202,19 +202,21 @@ def test_evaluate_cpu(self): # Check that opencv is used for pre-processing self.assertIn("pre-process-backend=opencv", result) - def test_evaluate_gpu(self): + def test_evaluate_gpu_0(self): result = self.pipeline.evaluate( constants=self.constants, parameters={ - "object_detection_device": "GPU.1", + "object_detection_device": "GPU.0", "object_detection_batch_size": 0, "object_detection_inference_interval": 1, "object_detection_nireq": 0, - "object_classification_device": "GPU.1", + "object_classification_device": "GPU.0", "object_classification_batch_size": 0, "object_classification_inference_interval": 1, "object_classification_nireq": 0, "object_classification_reclassify_interval": 1, + "compositor_device": "GPU.0", + "decoding_device": "GPU.0", "tracking_type": "short-term-imageless", "pipeline_watermark_enabled": True, "live_preview_enabled": False, @@ -242,6 +244,51 @@ def test_evaluate_gpu(self): # Check that va-surface-sharing is used for pre-processing self.assertIn("pre-process-backend=va-surface-sharing", result) + # Check that the right vaapi elements are used + self.assertIn("vacompositor", result) + + def test_evaluate_gpu_1(self): + result = self.pipeline.evaluate( + constants=self.constants, + parameters={ + "object_detection_device": "GPU.1", + "object_detection_batch_size": 0, + "object_detection_inference_interval": 1, + "object_detection_nireq": 0, + "object_classification_device": "GPU.1", + "object_classification_batch_size": 0, + "object_classification_inference_interval": 1, + "object_classification_nireq": 0, + "object_classification_reclassify_interval": 1, + "compositor_device": "GPU.1", + "decoding_device": "GPU.1", + "tracking_type": "short-term-imageless", + "pipeline_watermark_enabled": True, + "live_preview_enabled": False, + }, + regular_channels=self.regular_channels, + inference_channels=self.inference_channels, + elements=[ + ("va", "varenderD129compositor", "..."), + ("va", "varenderD129h264enc", "..."), + ("va", "varenderD129h264dec", "..."), + ("va", "varenderD129postproc", "..."), + ], + ) + + # Common checks + self.common_checks(result) + self.output_present_check(result) + self.sink_count_check(result) + self.gvaclassify_count_check(result, self.inference_channels) + + # Check that model proc is used + self.assertIn("model-proc=detection_model_proc.json", result) + self.assertIn("model-proc=classification_model_proc.json", result) + + # Check that va-surface-sharing is used for pre-processing + self.assertIn("pre-process-backend=va-surface-sharing", result) + # Check that the right vaapi elements are used self.assertIn("varenderD129compositor", result) @@ -272,10 +319,10 @@ def test_evaluate_no_model_proc(self): regular_channels=self.regular_channels, inference_channels=self.inference_channels, elements=[ - ("va", "vacompositor", "..."), - ("va", "vah264enc", "..."), - ("va", "vah264dec", "..."), - ("va", "vapostproc", "..."), + ("va", "compositor", "..."), + ("va", "x264enc", "..."), + ("va", "decodebin", "..."), + ("va", "videoscale", "..."), ], ) diff --git a/tools/visual-pipeline-and-platform-evaluation-tool/utils.py b/tools/visual-pipeline-and-platform-evaluation-tool/utils.py index a5cac4f30..93f4a6da8 100644 --- a/tools/visual-pipeline-and-platform-evaluation-tool/utils.py +++ b/tools/visual-pipeline-and-platform-evaluation-tool/utils.py @@ -62,6 +62,9 @@ def prepare_video_and_constants( "object_classification_reclassify_interval", 0.0 ) object_classification_nireq = kwargs.get("object_classification_nireq", 1) + compositor_device = str(kwargs.get("compositor_device", "")) + decoding_device = str(kwargs.get("decoding_device", "")) + tracking_type = kwargs.get("tracking_type", "short-term-imageless") pipeline_watermark_enabled = kwargs.get("pipeline_watermark_enabled", True) pipeline_video_enabled = kwargs.get("pipeline_video_enabled", True) @@ -95,6 +98,8 @@ def prepare_video_and_constants( object_classification_reclassify_interval ], "object_classification_nireq": [object_classification_nireq], + "compositor_device": compositor_device.split(", "), + "decoding_device": decoding_device.split(", "), "tracking_type": [tracking_type], "pipeline_watermark_enabled": [pipeline_watermark_enabled], "pipeline_video_enabled": [pipeline_video_enabled],