Debian Codegraph @

gst/tensordecoders/gstyolotensordecoder.c

raw
1 /*
2 * GStreamer gstreamer-yolotensordecoder
3 * Copyright (C) 2024 Collabora Ltd.
4 * Authors: Daniel Morin <[email protected]>
5 * Vineet Suryan <[email protected]>
6 * Santosh Mahto <[email protected]>
7 *
8 * gstyolotensordecoder.c
9 *
10 * This library is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Library General Public
12 * License as published by the Free Software Foundation; either
13 * version 2 of the License, or (at your option) any later version.
14 *
15 * This library is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * Library General Public License for more details.
19 *
20 * You should have received a copy of the GNU Library General Public
21 * License along with this library; if not, write to the
22 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
23 * Boston, MA 02110-1301, USA.
24 */
25
26 /**
27 * SECTION:element-yolotensordec
28 * @short_description: Decode tensors from YOLO detection models
29 *
30 * This element can parse per-buffer inference tensors meta data generated
31 * by an upstream inference element
32 *
33 *
34 * ## Example launch command:
35 *
36 * Test image file, model file and labels file can be found here :
37 * https://gitlab.collabora.com/gstreamer/onnx-models
38 *
39 * gst-launch-1.0 -v v4l2src \
40 * ! videoconvertscale qos=false ! video/x-raw, pixel-aspect-ratio=1/1 \
41 * ! onnxinference model-file=yolov8s.onnx \
42 * ! yolov8tensordec class-confidence-threshold=0.8 iou-threshold=0.3 \
43 * max-detections=100 label-file=labels/COCO_classes.txt \
44 * ! objectdetectionoverlay ! glimagesink sink=gtkglsink
45 *
46 * The original repository of the Yolo is located at
47 * https://github.com/ultralytics/ultralytics.
48 * For easy experimentation, the models based on Yolo architecture in Onnx
49 * format can be found at https://col.la/gstonnxmodels . This model already
50 * has tensors name embedded matching default values of tensors-detections-name
51 * and tensors-logits-name properties. It's also possible to embed tensor-ids
52 * into any model based on Yolo architecture to allow this tensor-decoder
53 * to decode tensors.This process is described in the Readme of
54 * repository: https://col.la/gstonnxmodels"
55 *
56 * Since: 1.28
57 */
58
59 #ifdef HAVE_CONFI_H
60 #include "config.h"
61 #endif
62
63 #include "gstyolotensordecoder.h"
64
65 #include <gst/analytics/analytics.h>
66 #include <gst/analytics/gstanalytics_image_util.h>
67 #include <gio/gio.h>
68
69 #include <math.h>
70
71 #define YOLO_DETECTION_MASK "yolo-v8-out"
72 GQuark YOLO_DETECTION_MASK_ID;
73
74 /**
75 * GstYoloTensorDecoder:
76 *
77 * A tensor decoder for YOLO v8-v11 models.
78 *
79 * Since: 1.28
80 */
81
82 GST_DEBUG_CATEGORY_STATIC (yolo_tensor_decoder_debug);
83 #define GST_CAT_DEFAULT yolo_tensor_decoder_debug
84
85 GST_ELEMENT_REGISTER_DEFINE (yolo_tensor_decoder, "yolov8tensordec",
86 GST_RANK_PRIMARY, GST_TYPE_YOLO_TENSOR_DECODER);
87
88 /* GstYoloTensorDecoder properties, see properties description in
89 * gst_yolo_tensor_decoder_class_init for more details. */
90 enum
91 {
92 PROP_0,
93 PROP_BOX_CONFI_THRESH,
94 PROP_CLS_CONFI_THRESH,
95 PROP_IOU_THRESH,
96 PROP_MAX_DETECTION,
97 PROP_LABEL_FILE
98 };
99
100 /* Specify the range of confidence level in tensor output*/
101 typedef struct _ConfidenceRange
102 {
103 gsize start; /* Start index of confidence level */
104 gsize end; /* End index of confidence level */
105 gsize step; /* Step size of next confidence level index */
106 } ConfidenceRange;
107
108 /* Default properties value */
109 static const gfloat DEFAULT_BOX_CONFI_THRESH = 0.4f;
110 static const gfloat DEFAULT_CLS_CONFI_THRESH = 0.4f;
111 static const gfloat DEFAULT_IOU_THRESH = 0.7f;
112 static const gsize DEFAULT_MAX_DETECTION = 100;
113
114 /* Global variable storing class for OD. Generally OD has class
115 * and we need to provide one but this class is just a placeholder.*/
116 GQuark OOI_CLASS_ID;
117
118 /* GStreamer element srcpad template. Template of a srcpad that can receive
119 * any raw video. */
120 static GstStaticPadTemplate gst_yolo_tensor_decoder_src_template =
121 GST_STATIC_PAD_TEMPLATE ("src",
122 GST_PAD_SRC,
123 GST_PAD_ALWAYS,
124 GST_STATIC_CAPS ("video/x-raw"));
125
126 /* *INDENT-OFF* */
127 static GstStaticPadTemplate gst_yolo_tensor_decoder_sink_template =
128 GST_STATIC_PAD_TEMPLATE ("sink",
129 GST_PAD_SINK,
130 GST_PAD_ALWAYS,
131 GST_STATIC_CAPS (
132 "video/x-raw,"
133 "tensors=(structure)["
134 "tensorgroups,"
135 "yolo-v8-out=(/uniquelist){"
136 "(GstCaps)["
137 "tensor/strided,"
138 "tensor-id=(string)yolo-v8-out,"
139 "dims=<(int)1,(int)[1,max],(int)[1,max]>,"
140 "dims-order=(string)col-major,"
141 "type=(string)float32"
142 "]"
143 "}"
144 "]"
145 ));
146 /* *INDENT-ON* */
147
148 /* GstYoloTensorDecoder Prototypes */
149 static gboolean gst_yolo_tensor_decoder_set_caps (GstBaseTransform * trans,
150 GstCaps * incaps, GstCaps * outcaps);
151 static void gst_yolo_tensor_decoder_set_property (GObject * object,
152 guint prop_id, const GValue * value, GParamSpec * pspec);
153 static void gst_yolo_tensor_decoder_get_property (GObject * object,
154 guint prop_id, GValue * value, GParamSpec * pspec);
155 static GstFlowReturn gst_yolo_tensor_decoder_transform_ip (GstBaseTransform *
156 trans, GstBuffer * buf);
157 static void gst_yolo_tensor_decoder_finalize (GObject * object);
158
159 static void gst_yolo_tensor_decoder_object_found (GstYoloTensorDecoder * self,
160 GstAnalyticsRelationMeta * rmeta, BBox * bb, gfloat confidence,
161 GQuark class_quark, const gfloat * candidate_masks, gsize offset,
162 guint count);
163
164 G_DEFINE_TYPE (GstYoloTensorDecoder, gst_yolo_tensor_decoder,
165 GST_TYPE_BASE_TRANSFORM);
166
167 static GArray *
168 read_labels (const char *labels_file)
169 {
170 GArray *array;
171 GFile *file = g_file_new_for_path (labels_file);
172 GFileInputStream *file_stream;
173 GDataInputStream *data_stream;
174 GError *error = NULL;
175 gchar *line;
176
177 file_stream = g_file_read (file, NULL, &error);
178 g_object_unref (file);
179 if (!file_stream) {
180 GST_WARNING ("Could not open file %s: %s\n", labels_file, error->message);
181 g_clear_error (&error);
182 return NULL;
183 }
184
185 data_stream = g_data_input_stream_new (G_INPUT_STREAM (file_stream));
186 g_object_unref (file_stream);
187
188 array = g_array_new (FALSE, FALSE, sizeof (GQuark));
189
190 while ((line = g_data_input_stream_read_line (data_stream, NULL, NULL,
191 &error))) {
192 GQuark label = g_quark_from_string (line);
193 g_array_append_val (array, label);
194 g_free (line);
195 }
196
197 g_object_unref (data_stream);
198
199 if (error) {
200 GST_WARNING ("Could not open file %s: %s", labels_file, error->message);
201 g_array_free (array, TRUE);
202 g_clear_error (&error);
203 return NULL;
204 }
205
206 if (array->len == 0) {
207 g_array_free (array, TRUE);
208 return NULL;
209 }
210
211 return array;
212 }
213
214 static gboolean
215 gst_yolo_tensor_decoder_set_caps (GstBaseTransform * trans, GstCaps * incaps,
216 GstCaps * outcaps)
217 {
218 GstYoloTensorDecoder *self = GST_YOLO_TENSOR_DECODER (trans);
219
220 if (!gst_video_info_from_caps (&self->video_info, incaps)) {
221 GST_ERROR_OBJECT (self, "Failed to parse caps");
222 return FALSE;
223 }
224
225 if (gst_base_transform_is_passthrough (trans)) {
226 GST_ERROR_OBJECT (self, "Failed. Can't handle passthrough");
227 return FALSE;
228 }
229
230 return TRUE;
231 }
232
233 static void
234 gst_yolo_tensor_decoder_set_property (GObject * object, guint prop_id,
235 const GValue * value, GParamSpec * pspec)
236 {
237 GstYoloTensorDecoder *self = GST_YOLO_TENSOR_DECODER (object);
238 const gchar *filename;
239
240 switch (prop_id) {
241 case PROP_BOX_CONFI_THRESH:
242 self->box_confi_thresh = g_value_get_float (value);
243 break;
244 case PROP_CLS_CONFI_THRESH:
245 self->cls_confi_thresh = g_value_get_float (value);
246 break;
247 case PROP_IOU_THRESH:
248 self->iou_thresh = g_value_get_float (value);
249 break;
250 case PROP_MAX_DETECTION:
251 self->max_detection = g_value_get_uint (value);
252 break;
253 case PROP_LABEL_FILE:
254 {
255 GArray *labels;
256
257 filename = g_value_get_string (value);
258 labels = read_labels (filename);
259
260 if (labels) {
261 g_free (self->label_file);
262 self->label_file = g_strdup (filename);
263 g_clear_pointer (&self->labels, g_array_unref);
264 self->labels = labels;
265 } else {
266 GST_WARNING_OBJECT (self, "Label file '%s' not found!", filename);
267 }
268 break;
269 }
270 default:
271 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
272 break;
273 }
274 }
275
276 static void
277 gst_yolo_tensor_decoder_get_property (GObject * object, guint prop_id,
278 GValue * value, GParamSpec * pspec)
279 {
280 GstYoloTensorDecoder *self = GST_YOLO_TENSOR_DECODER (object);
281
282 switch (prop_id) {
283 case PROP_BOX_CONFI_THRESH:
284 g_value_set_float (value, self->box_confi_thresh);
285 break;
286 case PROP_CLS_CONFI_THRESH:
287 g_value_set_float (value, self->cls_confi_thresh);
288 break;
289 case PROP_IOU_THRESH:
290 g_value_set_float (value, self->iou_thresh);
291 break;
292 case PROP_MAX_DETECTION:
293 g_value_set_uint (value, self->max_detection);
294 break;
295 case PROP_LABEL_FILE:
296 g_value_set_string (value, self->label_file);
297 break;
298 default:
299 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
300 break;
301 }
302 }
303
304 static void
305 gst_yolo_tensor_decoder_class_init (GstYoloTensorDecoderClass * klass)
306 {
307 GObjectClass *gobject_class = (GObjectClass *) klass;
308 GstElementClass *element_class = (GstElementClass *) klass;
309 GstBaseTransformClass *basetransform_class = (GstBaseTransformClass *) klass;
310
311 /* Define GstYoloTensorDecoder debug category. */
312 GST_DEBUG_CATEGORY_INIT (yolo_tensor_decoder_debug,
313 "yolov8tensordec", 0, "Tensor decoder for Yolo detection models");
314
315 YOLO_DETECTION_MASK_ID = g_quark_from_static_string (YOLO_DETECTION_MASK);
316
317 /* Set GObject vmethod to get and set property */
318 gobject_class->set_property = gst_yolo_tensor_decoder_set_property;
319 gobject_class->get_property = gst_yolo_tensor_decoder_get_property;
320
321 g_object_class_install_property (G_OBJECT_CLASS (klass),
322 PROP_BOX_CONFI_THRESH,
323 g_param_spec_float ("box-confidence-threshold",
324 "Box location confidence threshold",
325 "Boxes with a location confidence level inferior to this threshold "
326 "will be excluded",
327 0.0, 1.0, DEFAULT_BOX_CONFI_THRESH,
328 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
329
330 g_object_class_install_property (G_OBJECT_CLASS (klass),
331 PROP_CLS_CONFI_THRESH,
332 g_param_spec_float ("class-confidence-threshold",
333 "Class confidence threshold",
334 "Classes with a confidence level inferior to this threshold "
335 "will be excluded",
336 0.0, 1.0, DEFAULT_CLS_CONFI_THRESH,
337 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
338
339 g_object_class_install_property (G_OBJECT_CLASS (klass),
340 PROP_IOU_THRESH,
341 g_param_spec_float ("iou-threshold",
342 "Maximum IOU threshold",
343 "Maximum intersection-over-union between bounding boxes to "
344 "consider them distinct.",
345 0.0, 1.0, DEFAULT_IOU_THRESH,
346 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
347
348 g_object_class_install_property (G_OBJECT_CLASS (klass),
349 PROP_MAX_DETECTION,
350 g_param_spec_uint ("max-detections",
351 "Maximum object/masks detections.",
352 "Maximum object/masks detections.",
353 1, G_MAXUINT, DEFAULT_MAX_DETECTION,
354 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
355
356 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_LABEL_FILE,
357 g_param_spec_string ("label-file",
358 "Label file", "Label file", NULL, (GParamFlags)
359 (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
360
361
362 /* Element description. */
363 gst_element_class_set_static_metadata (element_class,
364 "YOLO v8-11 object detection tensor decoder", "Tensordecoder/Video",
365 "Decode tensors output from the inference of YOLO Object Detection or FastSAM model (Detection)"
366 "on video frames. This works on YOLO version 8 and later(v11), and FastSAM models.",
367 "Daniel Morin <[email protected]>, Santosh Mahto <[email protected]>");
368
369 /* Add pads to element base on pad template defined earlier */
370 gst_element_class_add_pad_template (element_class,
371 gst_static_pad_template_get (&gst_yolo_tensor_decoder_src_template));
372 gst_element_class_add_pad_template (element_class,
373 gst_static_pad_template_get (&gst_yolo_tensor_decoder_sink_template));
374
375 /* Set GstBaseTransform vmethod transform_ip. This methode is called
376 * by the srcpad when it receive buffer. ip stand for in-place meaning the
377 * buffer remain unchanged by the element. Tensor-decoder only monitor
378 * buffer it receive for a meta attach to the buffer that is a GstTensorMeta
379 * and has a tensor-id can be handled by GstYoloTensorDecoder. */
380 basetransform_class->transform_ip =
381 GST_DEBUG_FUNCPTR (gst_yolo_tensor_decoder_transform_ip);
382
383 /* Set GstBaseTransform set_caps vmethod. This will be called once the
384 * capability negotiation has been completed. We will be able to extract
385 * resolution from this callback. */
386 basetransform_class->set_caps =
387 GST_DEBUG_FUNCPTR (gst_yolo_tensor_decoder_set_caps);
388
389 gobject_class->finalize = gst_yolo_tensor_decoder_finalize;
390
391 klass->object_found = gst_yolo_tensor_decoder_object_found;
392
393 /* Calculate the class id placeholder (also a quark) that will be set
394 * as label if object if labels are not provided via label-file. */
395 OOI_CLASS_ID = g_quark_from_static_string ("Yolo-None");
396 }
397
398 struct Candidate
399 {
400 const float *candidate;
401 const float max_confidence;
402 const guint max_class_offset;
403 };
404
405 static void
406 gst_yolo_tensor_decoder_init (GstYoloTensorDecoder * self)
407 {
408 /* GstYoloTensorDecoder instance initialization */
409 self->box_confi_thresh = DEFAULT_BOX_CONFI_THRESH;
410 self->cls_confi_thresh = DEFAULT_CLS_CONFI_THRESH;
411 self->iou_thresh = DEFAULT_IOU_THRESH;
412 self->max_detection = DEFAULT_MAX_DETECTION;
413
414 self->sel_candidates = g_array_new (FALSE, FALSE, sizeof (struct Candidate));
415 self->selected = g_ptr_array_new ();
416
417 gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (self), FALSE);
418 }
419
420 static const GstTensor *
421 gst_yolo_tensor_decoder_get_tensor (GstYoloTensorDecoder *
422 self, GstBuffer * buf)
423 {
424 GstMeta *meta = NULL;
425 gpointer iter_state = NULL;
426
427 if (!gst_buffer_get_meta (buf, GST_TENSOR_META_API_TYPE)) {
428 GST_DEBUG_OBJECT (self,
429 "missing tensor meta from buffer %" GST_PTR_FORMAT, buf);
430 return NULL;
431 }
432
433 while ((meta = gst_buffer_iterate_meta_filtered (buf, &iter_state,
434 GST_TENSOR_META_API_TYPE))) {
435 GstTensorMeta *tensor_meta = (GstTensorMeta *) meta;
436 const GstTensor *tensor;
437 const gsize YOLO_DETECTIONS_TENSOR_N_DIMS = 3;
438 static const gsize dims[3] = { 1, G_MAXSIZE, G_MAXSIZE };
439
440 tensor = gst_tensor_meta_get_typed_tensor (tensor_meta,
441 YOLO_DETECTION_MASK_ID, GST_TENSOR_DATA_TYPE_FLOAT32,
442 GST_TENSOR_DIM_ORDER_COL_MAJOR, YOLO_DETECTIONS_TENSOR_N_DIMS, dims);
443
444 if (tensor) {
445 if (tensor->dims[1] < 5) {
446 GST_WARNING_OBJECT (self, "Ignore tensor because dims[1] is %zu < 5",
447 tensor->dims[1]);
448 continue;
449 }
450
451 return tensor;
452 }
453 }
454
455 return NULL;
456 }
457
458
459 static GstFlowReturn
460 gst_yolo_tensor_decoder_transform_ip (GstBaseTransform * trans, GstBuffer * buf)
461 {
462 GstYoloTensorDecoder *self = GST_YOLO_TENSOR_DECODER (trans);
463 GstAnalyticsRelationMeta *rmeta;
464 const GstTensor *detections_tensor;
465
466 detections_tensor = gst_yolo_tensor_decoder_get_tensor (self, buf);
467 if (detections_tensor == NULL) {
468 GST_WARNING_OBJECT (self, "Couldn't find mask tensor, skipping");
469 return GST_FLOW_OK;
470 }
471
472 /* Retrieve or attach an analytics-relation-meta to the buffer.
473 * Analytics-relation-meta are container that can reveive multiple
474 * analytics-meta, like OD and Segmentation. The following call will only
475 * retrieve an analytics-relation-meta if it exist or create one if it
476 * does not exist. */
477 rmeta = gst_buffer_add_analytics_relation_meta (buf);
478 if (rmeta == NULL) {
479 GST_ELEMENT_ERROR (trans, STREAM, FAILED, (NULL),
480 ("Analytics Relation meta allocation failed"));
481 return GST_FLOW_ERROR;
482 }
483
484 if (!gst_yolo_tensor_decoder_decode_f32 (self, rmeta, detections_tensor, 0))
485 return GST_FLOW_ERROR;
486
487 return GST_FLOW_OK;
488 }
489
490 static void
491 gst_yolo_tensor_decoder_finalize (GObject * object)
492 {
493 GstYoloTensorDecoder *self = GST_YOLO_TENSOR_DECODER (object);
494
495 g_clear_pointer (&self->sel_candidates, g_array_unref);
496 g_clear_pointer (&self->selected, g_ptr_array_unref);
497
498 g_free (self->label_file);
499 g_clear_pointer (&self->labels, g_array_unref);
500
501 G_OBJECT_CLASS (gst_yolo_tensor_decoder_parent_class)->finalize (object);
502 }
503
504 /* Extract bounding box from tensor data */
505 static void
506 gst_yolo_tensor_decoder_convert_bbox (const gfloat * candidate,
507 const gsize * offset, BBox * bbox)
508 {
509 gfloat w = *(candidate + offset[2]);
510 gfloat h = *(candidate + offset[3]);
511 bbox->x = *(candidate + offset[0]) - (w / 2);
512 bbox->y = *(candidate + offset[1]) - (h / 2);
513 bbox->w = w + 0.5;
514 bbox->h = h + 0.5;
515 }
516
517 /* Calculate iou between boundingbox of candidate c1 and c2
518 */
519 static gfloat
520 gst_yolo_tensor_decoder_iou (const gfloat * c1, const gfloat * c2,
521 const gsize * offset, BBox * bb1, BBox * bb2)
522 {
523 gst_yolo_tensor_decoder_convert_bbox (c1, offset, bb1);
524 gst_yolo_tensor_decoder_convert_bbox (c2, offset, bb2);
525 return gst_analytics_image_util_iou_int (bb1->x, bb1->y, bb1->w, bb1->h,
526 bb2->x, bb2->y, bb2->w, bb2->h);
527 }
528
529 /* Utility function to find maxmum confidence value across classes
530 * specified by range.
531 */
532 static gfloat
533 gst_yolo_tensor_decoder_find_max_class_confidence (const gfloat * c,
534 const ConfidenceRange * c_range, gsize * max_class_ofs)
535 {
536 gfloat max_val = 0.0;
537 for (gsize i = c_range->start; i <= c_range->end; i += c_range->step) {
538 if (*(c + i) > max_val) {
539 max_val = *(c + i);
540 *max_class_ofs = i;
541 }
542 }
543 return max_val;
544 }
545
546 /* Compare c1 and c2
547 * Utility function for sorting candiates based on the a field identified
548 * by offset.
549 */
550 static gint
551 gst_yolo_tensor_decoder_sort_candidates (gconstpointer p1, gconstpointer p2)
552 {
553 const struct Candidate *c1 = p1;
554 const struct Candidate *c2 = p2;
555
556 if (c1->max_confidence < c2->max_confidence)
557 return 1;
558 else if (c1->max_confidence > c2->max_confidence)
559 return -1;
560 else
561 return 0;
562 }
563
564 static gboolean
565 gst_yolo_tensor_decoder_decode_valid_bb (GstYoloTensorDecoder * self,
566 gfloat x, gfloat y, gfloat w, gfloat h)
567 {
568 GstYoloTensorDecoder *parent = GST_YOLO_TENSOR_DECODER (self);
569
570 if (x > (GST_VIDEO_INFO_WIDTH (&parent->video_info)))
571 return FALSE;
572 if (y > (GST_VIDEO_INFO_HEIGHT (&parent->video_info)))
573 return FALSE;
574 if (x < -(gfloat) (GST_VIDEO_INFO_WIDTH (&parent->video_info) / 2.0))
575 return FALSE;
576 if (y < -(gfloat) (GST_VIDEO_INFO_HEIGHT (&parent->video_info) / 2.0))
577 return FALSE;
578 if (w <= 0)
579 return FALSE;
580 if (h <= 0)
581 return FALSE;
582 if (w > (GST_VIDEO_INFO_WIDTH (&parent->video_info)))
583 return FALSE;
584 if (h > (GST_VIDEO_INFO_HEIGHT (&parent->video_info)))
585 return FALSE;
586
587 return TRUE;
588 }
589
590 gboolean
591 gst_yolo_tensor_decoder_decode_f32 (GstYoloTensorDecoder * self,
592 GstAnalyticsRelationMeta * rmeta, const GstTensor * detections_tensor,
593 guint num_masks)
594 {
595 GstMapInfo map_info_detections;
596 gfloat iou;
597 gboolean rv, keep;
598 gsize offset, x_offset, y_offset, w_offset, h_offset, offsets[4];
599 BBox bb1, bb2;
600 ConfidenceRange c_range;
601 gsize max_class_offset = 0, class_index;
602 GQuark class_quark = OOI_CLASS_ID;
603 gboolean ret = TRUE;
604 gsize i;
605
606 /* Retrieve memory at index 0 and map it in READWRITE mode */
607 rv = gst_buffer_map (detections_tensor->data, &map_info_detections,
608 GST_MAP_READ);
609 if (rv == FALSE) {
610 GST_ELEMENT_ERROR (self, STREAM, FAILED, (NULL),
611 ("Could not map tensor buffer %" GST_PTR_FORMAT,
612 detections_tensor->data));
613 return FALSE;
614 }
615
616 GST_LOG_OBJECT (self, "Mask Tensor shape dims %zu",
617 detections_tensor->num_dims);
618
619 /* Trace detections tensor dimensions */
620 if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_TRACE) {
621 for (gsize i = 0; i < detections_tensor->num_dims; i++) {
622 GST_TRACE_OBJECT (self, "Detections Tensor dim %zu: %zu", i,
623 detections_tensor->dims[i]);
624 }
625 }
626
627 /* Number of candidates can be large, reset the arrays */
628 g_array_set_size (self->sel_candidates, 0);
629 g_ptr_array_set_size (self->selected, 0);
630
631 /* detections_tensor->dims[2] contain the number of candidates. Let's call the
632 * number of candidates C. We store this value in offset as we use it
633 * calculate the offset of candidate fields. The variable #data_detections above point
634 * at the detections tensor data, but candidates data is organize like a plane.
635 * Candidates bbox X coord fields from 0 to C start at the begining of the
636 * tensor data and are continguous in memory, followed by all candidates
637 * field Y, followed by field W, ... followed by field class confidence level,
638 * ..., followed by all candidates mask0, ..., followed by all candidates
639 * mask31. Bellow we pre-calculate each field offset relative to the
640 * candidate pointer (pointer to field X), which will allow us to easily
641 * access each candiates field.
642 * */
643 offset = detections_tensor->dims[2];
644 x_offset = 0;
645 y_offset = offset;
646 w_offset = 2 * offset;
647 h_offset = 3 * offset;
648 /* first index that contain confidence level */
649 c_range.start = 4 * offset;
650 /* Last index that contains confidence level */
651 c_range.end = (detections_tensor->dims[1] - num_masks - 1) * offset;
652 /* Step between class confidence level */
653 c_range.step = offset;
654 offsets[0] = x_offset;
655 offsets[1] = y_offset;
656 offsets[2] = w_offset;
657 offsets[3] = h_offset;
658
659 #define BB_X(candidate) candidate[x_offset]
660 #define BB_Y(candidate) candidate[y_offset]
661 #define BB_W(candidate) candidate[w_offset]
662 #define BB_H(candidate) candidate[h_offset]
663
664 for (gsize c_idx = 0; c_idx < detections_tensor->dims[2]; c_idx++) {
665 float *candidate = (float *) map_info_detections.data;
666
667 candidate += c_idx;
668
669 /* Yolo have multiple class, so maximum confidence level across all class is used
670 * to evaluate the relevance of the candidate. Here we filter candidates
671 * based on their class confidence level.*/
672 gfloat max_confidence =
673 gst_yolo_tensor_decoder_find_max_class_confidence (candidate, &c_range,
674 &max_class_offset);
675 if (max_confidence > self->cls_confi_thresh
676 && gst_yolo_tensor_decoder_decode_valid_bb (self,
677 BB_X (candidate), BB_Y (candidate), BB_W (candidate),
678 BB_H (candidate))) {
679
680 struct Candidate c = {
681 candidate,
682 max_confidence,
683 max_class_offset,
684 };
685 g_array_append_val (self->sel_candidates, c);
686
687 GST_TRACE_OBJECT (self, "%zu: x,y=(%f;%f) w,h=(%f;%f), s=%f c=%f",
688 c_idx, candidate[x_offset], candidate[y_offset],
689 candidate[w_offset], candidate[h_offset],
690 candidate[w_offset] * candidate[h_offset], max_confidence);
691 }
692
693 /* Pointer arithmetic, going to the next candidate. This is the candidate
694 * pointer that is now incremented to the next candidate which is also
695 * the field X of the next candidate.*/
696 candidate += 1;
697 }
698
699 GST_LOG_OBJECT (self, "Before NMS selected candidates count: %u",
700 self->sel_candidates->len);
701
702 /* We sort the remaining candidates because, in the next selection phase we
703 * have a maximum and we want to make sure that considered only the candidates
704 * with the highest class confidence level before potentially reaching the
705 * maximum.*/
706 g_array_sort (self->sel_candidates, gst_yolo_tensor_decoder_sort_candidates);
707
708 if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_TRACE) {
709 for (i = 0; i < self->sel_candidates->len; i++) {
710 struct Candidate *c = &g_array_index (self->sel_candidates,
711 struct Candidate, i);
712 GST_TRACE_OBJECT (self,
713 "Sorted: %zu: x,y=(%f;%f) w,h=(%f;%f), s=%f c=%f",
714 i,
715 c->candidate[x_offset],
716 c->candidate[y_offset],
717 c->candidate[w_offset],
718 c->candidate[h_offset],
719 c->candidate[w_offset] * c->candidate[h_offset], c->max_confidence);
720 }
721 }
722
723 /* Algorithm in part inspired by OpenCV NMSBoxes */
724 for (i = 0; i < self->sel_candidates->len; i++) {
725 const struct Candidate *c = &g_array_index (self->sel_candidates,
726 struct Candidate, i);
727 keep = TRUE;
728
729 /* We only want to a NMS using IoU between candidates we've decided to
730 * keep and the new one we considering to keep. selected array contain
731 * the candidates we decided to keep and candidates[c] is the candidate
732 * we're considering to keep or reject */
733 for (gsize s = 0; s < self->selected->len && keep; s++) {
734 const float *candidate2 = g_ptr_array_index (self->selected, s);
735 iou = gst_yolo_tensor_decoder_iou (c->candidate, candidate2,
736 offsets, &bb1, &bb2);
737 keep = (iou <= self->iou_thresh);
738 }
739
740 if (keep) {
741 if (self->selected->len == 0) {
742 /* The first bounding-box always get in as there's no others bbox
743 * to filter on based on IoU */
744 gst_yolo_tensor_decoder_convert_bbox (c->candidate, offsets, &bb1);
745 }
746
747 g_ptr_array_add (self->selected, (gpointer) c->candidate);
748
749 if (self->labels) {
750 class_index = (c->max_class_offset - c_range.start) / c_range.step;
751
752 if (class_index < self->labels->len)
753 class_quark = g_array_index (self->labels, GQuark, class_index);
754 }
755
756 const gfloat *candidate_masks = NULL;
757 if (num_masks) {
758 /* detections weight will be stored in last `num_masks`
759 * row of detections_tensor, so mask offset
760 * will start at the end of the detections_tensor minus
761 * `num_masks`
762 */
763 candidate_masks = c->candidate +
764 ((detections_tensor->dims[1] - num_masks) * offset);
765
766 if (candidate_masks + num_masks + offset >
767 (gfloat *) (map_info_detections.data + map_info_detections.size)) {
768 GST_ELEMENT_ERROR (self, STREAM, FAILED, (NULL),
769 ("Mask tensor data size %zu is smaller than expected (%zu)",
770 (candidate_masks - (gfloat *) map_info_detections.data) +
771 offset + num_masks, map_info_detections.size));
772 ret = FALSE;
773 break;
774 }
775 }
776
777 GST_YOLO_TENSOR_DECODER_GET_CLASS (self)->object_found (self, rmeta, &bb1,
778 c->max_confidence, class_quark, candidate_masks, offset,
779 self->selected->len);
780
781 /* If the maximum number of candidate selected is reached exit the
782 * selection process. */
783 if (self->selected->len >= self->max_detection) {
784 break;
785 }
786 }
787 }
788
789 GST_LOG_OBJECT (self, "After NMS selected count: %u", self->selected->len);
790
791 /* We unmap the memory */
792 gst_buffer_unmap (detections_tensor->data, &map_info_detections);
793
794 return ret;
795 }
796
797 static void
798 gst_yolo_tensor_decoder_object_found (GstYoloTensorDecoder * self,
799 GstAnalyticsRelationMeta * rmeta, BBox * bb, gfloat confidence,
800 GQuark class_quark, const gfloat * candidate_masks, gsize offset,
801 guint count)
802 {
803 gst_analytics_relation_meta_add_od_mtd (rmeta, class_quark, bb->x, bb->y,
804 bb->w, bb->h, confidence, NULL);
805 }