Debian Codegraph @

gst/tensordecoders/gstfacedetectortensordecoder.c

raw
1 /*
2 * GStreamer gstreamer-ultralightfacedetectortensordec
3 * Copyright (C) 2025 Collabora Ltd.
4 *
5 * gstfacedetectortensordecoder.c
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
16 *
17 * You should have received a copy of the GNU Library General Public
18 * License along with this library; if not, write to the
19 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
20 * Boston, MA 02110-1301, USA.
21 */
22
23 /**
24 * SECTION:element-ultralightfacedetectortensordec
25 * @short_description: Detect faces in video buffers using the Ultra Light Face Detection model.
26 *
27 * This element can parse per-buffer inference tensor meta data generated by an upstream
28 * inference element.
29 *
30 * ## Example launch command:
31 *
32 * Test image files can be found here :
33 * https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB/tree/master/imgs
34 *
35 * The Model file can be found here :
36 * https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB/tree/master/models/onnx
37 *
38 * GST_DEBUG=ultralightfacedetectortensordec \
39 * gst-launch-1.0 multifilesrc location=~/imgs/11.jpg ! jpegdec ! videoconvertscale ! \
40 * onnxinference model-file=version-RFB-320.onnx input-image-format=chw input-tensor-offset=-127 input-tensor-scale=128.0 ! \
41 * ultralightfacedetectortensordec ! objectdetectionoverlay object-detection-outline-color=0xFF0000FF draw-labels=false ! \
42 * videoconvertscale ! autovideosink
43 *
44 * Since: 1.28
45 */
46
47 #ifdef HAVE_CONFIG_H
48 #include "config.h"
49 #endif
50
51 #include "gstfacedetectortensordecoder.h"
52
53 #include <gio/gio.h>
54
55 #include <gst/gst.h>
56 #include <gst/video/video.h>
57 #include <gst/analytics/analytics.h>
58
59 /* Face detection tensor id strings */
60 #define BOXES_TENSOR_ID "ssd-mobilenet-v1-variant-1-out-boxes"
61 #define SCORES_TENSOR_ID "ultra-lightweight-face-detection-rfb-320-v1-variant-1-out-scores"
62 #define GROUP_ID "ultra-lightweight-face-detection-rfb-320-v1-variant-1-out"
63
64 GST_DEBUG_CATEGORY_STATIC (face_detector_tensor_decoder_debug);
65 #define GST_CAT_DEFAULT face_detector_tensor_decoder_debug
66
67 GST_ELEMENT_REGISTER_DEFINE (face_detector_tensor_decoder,
68 "ultralightfacedetectortensordec", GST_RANK_PRIMARY,
69 GST_TYPE_FACE_DETECTOR_TENSOR_DECODER);
70
71 /* GstFaceDetectorTensorDecoder properties, see properties description in
72 * gst_face_detector_tensor_decoder_class_init for more details. */
73 enum
74 {
75 PROP_0,
76 PROP_SCORE_THRESHOLD,
77 PROP_IOU_THRESHOLD
78 };
79
80 /* Default properties value */
81 static const gfloat DEFAULT_SCORE_THRESHOLD = 0.6f; /* confidence threshold */
82 static const gfloat DEFAULT_IOU_THRESHOLD = 0.3f; /* NMS IoU threshold */
83
84 /* To tensor-id are defined by a string that is converted to quark
85 * which is just an integer value using a hash function. For efficiency
86 * we compare on the quark (hash value). Since tensor-id never change we
87 * just calculate the hash once during initialization and store the value in
88 * these variables. */
89 GQuark BOXES_TENSOR_ID_QUARK;
90 GQuark SCORES_TENSOR_ID_QUARK;
91
92 GQuark FACE_QUARK;
93
94 /* GStreamer element srcpad template. Template of a srcpad that can receive
95 * any raw video. */
96 static GstStaticPadTemplate gst_face_detector_tensor_decoder_src_template =
97 GST_STATIC_PAD_TEMPLATE ("src",
98 GST_PAD_SRC,
99 GST_PAD_ALWAYS,
100 GST_STATIC_CAPS ("video/x-raw")
101 );
102
103 /* GStreamer element sinkpad template. Template of a sinkpad that can receive
104 * any raw video. */
105
106 /* *INDENT-OFF* */
107 static GstStaticPadTemplate gst_face_detector_tensor_decoder_sink_template =
108 GST_STATIC_PAD_TEMPLATE ("sink",
109 GST_PAD_SINK,
110 GST_PAD_ALWAYS,
111 GST_STATIC_CAPS ("video/x-raw,"
112 "tensors=(structure)["
113 "tensorgroups,"
114 GROUP_ID"=(/uniquelist){"
115 "(GstCaps)["
116 "tensor/strided,"
117 "tensor-id="BOXES_TENSOR_ID","
118 "dims=<(int)1,(int)[1,max],(int)4>,"
119 "dims-order=(string)row-major,"
120 "type=(string)float32;],"
121 "(GstCaps)["
122 "tensor/strided,"
123 "tensor-id="SCORES_TENSOR_ID","
124 "dims=<(int)1,(int)[1,max],(int)2>,"
125 "dims-order=(string)row-major,"
126 "type=(string)float32;]"
127 "}"
128 "];"
129 ));
130 /* *INDENT-ON* */
131
132 /* Prototypes */
133 static void gst_face_detector_tensor_decoder_set_property (GObject * object,
134 guint prop_id, const GValue * value, GParamSpec * pspec);
135 static void gst_face_detector_tensor_decoder_get_property (GObject * object,
136 guint prop_id, GValue * value, GParamSpec * pspec);
137 static void gst_face_detector_tensor_decoder_finalize (GObject * object);
138 static GstFlowReturn
139 gst_face_detector_tensor_decoder_transform_ip (GstBaseTransform * trans,
140 GstBuffer * buf);
141 static gboolean gst_face_detector_tensor_decoder_set_caps (GstBaseTransform *
142 trans, GstCaps * incaps, GstCaps * outcaps);
143
144 G_DEFINE_TYPE (GstFaceDetectorTensorDecoder, gst_face_detector_tensor_decoder,
145 GST_TYPE_BASE_TRANSFORM);
146
147 static void
148 gst_face_detector_tensor_decoder_class_init (GstFaceDetectorTensorDecoderClass
149 * klass)
150 {
151 GObjectClass *gobject_class = (GObjectClass *) klass;
152 GstElementClass *element_class = (GstElementClass *) klass;
153 GstBaseTransformClass *basetransform_class = (GstBaseTransformClass *) klass;
154
155 /* Define GstFaceDetectorTensorDecoder debug category. */
156 GST_DEBUG_CATEGORY_INIT (face_detector_tensor_decoder_debug,
157 "ultralightfacedetectortensordec", 0,
158 "Tensor Decoder for Face Detection");
159
160 /* Set GObject vmethod to get and set property */
161 gobject_class->set_property = gst_face_detector_tensor_decoder_set_property;
162 gobject_class->get_property = gst_face_detector_tensor_decoder_get_property;
163 gobject_class->finalize = gst_face_detector_tensor_decoder_finalize;
164
165 /* Define GstFaceDetectorTensorDecoder properties using GObject properties
166 * interface.*/
167
168 /**
169 * GstFaceDetectorTensorDecoder:score-threshold
170 *
171 * Threshold for deciding when to remove boxes based on score
172 *
173 * Since: 1.28
174 */
175 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_SCORE_THRESHOLD,
176 g_param_spec_float ("score-threshold",
177 "Score threshold",
178 "Threshold for deciding when to remove boxes based on score",
179 0.0, 1.0, DEFAULT_SCORE_THRESHOLD, (GParamFlags)
180 (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
181
182 /**
183 * GstFaceDetectorTensorDecoder:iou-threshold
184 *
185 * Threshold for removing boxes based on proportion of the image
186 *
187 * Since: 1.28
188 */
189 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_IOU_THRESHOLD,
190 g_param_spec_float ("iou-threshold",
191 "IoU threshold",
192 "Threshold for removing boxes based on proportion of the image",
193 0.0, 1.0, DEFAULT_IOU_THRESHOLD,
194 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
195
196 /* Element description. */
197 gst_element_class_set_static_metadata (element_class,
198 "ultralightfacedetectortensordec", "Tensordecoder/Video",
199 "Detect tensor output from the inference of Ultra Light Face Detection"
200 " to detect the faces in video frames."
201 "The original repository of the Ultra Light Face Detection is located at"
202 " https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB.",
203 "Raghavendra Rao <[email protected]>");
204
205 /* Add pads to element base on pad template defined earlier */
206
207 gst_element_class_add_pad_template (element_class,
208 gst_static_pad_template_get
209 (&gst_face_detector_tensor_decoder_sink_template));
210
211 gst_element_class_add_pad_template (element_class,
212 gst_static_pad_template_get
213 (&gst_face_detector_tensor_decoder_src_template));
214
215 /* Set GstBaseTransform vmethod transform_ip. This methode is called
216 * by the srcpad when it receive buffer. ip stand for in-place meaning the
217 * buffer remain unchanged by the element. Tensor-decoder only monitor
218 * buffer it receive for a meta attach to the buffer that is a GstTensorMeta
219 * and has a tensor-id can be handled by GstFaceDetectorTensorDecoder. */
220 basetransform_class->transform_ip =
221 GST_DEBUG_FUNCPTR (gst_face_detector_tensor_decoder_transform_ip);
222
223 /* Set GstBaseTransform set_caps vmethod. This will be called once the
224 * capability negotiation has been completed. We will be able to extract
225 * resolution from this callback. */
226 basetransform_class->set_caps =
227 GST_DEBUG_FUNCPTR (gst_face_detector_tensor_decoder_set_caps);
228
229 BOXES_TENSOR_ID_QUARK = g_quark_from_static_string (BOXES_TENSOR_ID);
230 SCORES_TENSOR_ID_QUARK = g_quark_from_static_string (SCORES_TENSOR_ID);
231 FACE_QUARK = g_quark_from_static_string ("face");
232 }
233
234 static void
235 gst_face_detector_tensor_decoder_init (GstFaceDetectorTensorDecoder * self)
236 {
237 self->score_threshold = DEFAULT_SCORE_THRESHOLD;
238 self->iou_threshold = DEFAULT_IOU_THRESHOLD;
239 self->sel_candidates = NULL;
240 self->selected = NULL;
241 gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (self), FALSE);
242
243 GST_PAD_UNSET_ACCEPT_INTERSECT (self->basetransform.sinkpad);
244 }
245
246 static void
247 gst_face_detector_tensor_decoder_finalize (GObject * object)
248 {
249 GstFaceDetectorTensorDecoder *self =
250 GST_FACE_DETECTOR_TENSOR_DECODER (object);
251
252 g_clear_pointer (&self->sel_candidates, g_ptr_array_unref);
253 g_clear_pointer (&self->selected, g_ptr_array_unref);
254 g_clear_pointer (&self->candidates, g_free);
255
256 G_OBJECT_CLASS (gst_face_detector_tensor_decoder_parent_class)->finalize
257 (object);
258 }
259
260 static void
261 gst_face_detector_tensor_decoder_set_property (GObject * object, guint prop_id,
262 const GValue * value, GParamSpec * pspec)
263 {
264 GstFaceDetectorTensorDecoder *self =
265 GST_FACE_DETECTOR_TENSOR_DECODER (object);
266
267 switch (prop_id) {
268 case PROP_SCORE_THRESHOLD:
269 self->score_threshold = g_value_get_float (value);
270 break;
271 case PROP_IOU_THRESHOLD:
272 self->iou_threshold = g_value_get_float (value);
273 break;
274 default:
275 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
276 break;
277 }
278 }
279
280 static void
281 gst_face_detector_tensor_decoder_get_property (GObject * object, guint prop_id,
282 GValue * value, GParamSpec * pspec)
283 {
284 GstFaceDetectorTensorDecoder *self =
285 GST_FACE_DETECTOR_TENSOR_DECODER (object);
286
287 switch (prop_id) {
288 case PROP_SCORE_THRESHOLD:
289 g_value_set_float (value, self->score_threshold);
290 break;
291 case PROP_IOU_THRESHOLD:
292 g_value_set_float (value, self->iou_threshold);
293 break;
294 default:
295 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
296 break;
297 }
298 }
299
300 /* gst_face_detector_tensor_decoder_set_caps:
301 *
302 * Callback on caps negotiation completed. We use it here to retrieve
303 * video resolution. See GstBaseTransform for more details.
304 */
305 static gboolean
306 gst_face_detector_tensor_decoder_set_caps (GstBaseTransform * trans,
307 GstCaps * incaps, GstCaps * outcaps)
308 {
309 GstFaceDetectorTensorDecoder *self = GST_FACE_DETECTOR_TENSOR_DECODER (trans);
310
311 if (!gst_video_info_from_caps (&self->video_info, incaps)) {
312 GST_ERROR_OBJECT (self, "Failed to parse caps");
313 return FALSE;
314 }
315
316 return TRUE;
317 }
318
319 /* gst_face_detector_tensor_decoder_get_tensor_meta
320 * @buf:in: buffer
321 * @boxes_tensor:out: Boxes tensor
322 * @scores_tensor:out: scores tensor
323 *
324 * Retrieve FaceDetection boxes and scores tensors from buffer.
325 *
326 * @return: TRUE if buf has boxes and scores tensor with desired features are attached to it.
327 * Otherwise FALSE will be returned.
328 */
329 static gboolean
330 gst_face_detector_tensor_decoder_get_tensor_meta (GstFaceDetectorTensorDecoder
331 * self, GstBuffer * buf, const GstTensor ** boxes_tensor,
332 const GstTensor ** scores_tensor)
333 {
334 GstMeta *meta;
335 gpointer state = NULL;
336 static const gsize BOXES_DIMS[] = { 1, G_MAXSIZE, 4 };
337 static const gsize SCORES_DIMS[] = { 1, G_MAXSIZE, 2 };
338
339 g_return_val_if_fail (boxes_tensor != NULL, FALSE);
340 g_return_val_if_fail (scores_tensor != NULL, FALSE);
341
342 *boxes_tensor = NULL;
343 *scores_tensor = NULL;
344
345 /* Find ultralightfacedetectortensordec TensorMeta */
346 while ((meta = gst_buffer_iterate_meta_filtered (buf, &state,
347 GST_TENSOR_META_API_TYPE))) {
348 GstTensorMeta *tensor_meta = (GstTensorMeta *) meta;
349
350 GST_LOG_OBJECT (self, "Num tensors %zu", tensor_meta->num_tensors);
351
352 /* Retrieve the tensor that has a tensor-id matching
353 * BOXES_TENSOR_ID_QUARK in the GstTensorMeta along with
354 * the reading order from the memory matching with GST_TENSOR_DIM_ORDER_ROW_MAJOR,
355 * 3 dimensions and the data type matching with GST_TENSOR_DATA_TYPE_FLOAT32 */
356 *boxes_tensor =
357 gst_tensor_meta_get_typed_tensor (tensor_meta, BOXES_TENSOR_ID_QUARK,
358 GST_TENSOR_DATA_TYPE_FLOAT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 3,
359 BOXES_DIMS);
360
361 if (*boxes_tensor == NULL)
362 continue;
363
364 /* Retrieve the tensor that has a tensor-id matching
365 * SCORES_TENSOR_ID_QUARK in the GstTensorMeta along with
366 * the reading order from the memory matching with GST_TENSOR_DIM_ORDER_ROW_MAJOR,
367 * 3 dimensions and the data type matching with GST_TENSOR_DATA_TYPE_FLOAT32 */
368 *scores_tensor =
369 gst_tensor_meta_get_typed_tensor (tensor_meta, SCORES_TENSOR_ID_QUARK,
370 GST_TENSOR_DATA_TYPE_FLOAT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 3,
371 SCORES_DIMS);
372
373 if (*scores_tensor == NULL)
374 continue;
375
376 }
377
378 if (*boxes_tensor == NULL) {
379 GST_WARNING_OBJECT (self, "Can't retrieve boxes tensor");
380 return FALSE;
381 }
382
383 if (*scores_tensor == NULL) {
384 GST_WARNING_OBJECT (self, "Can't retrieve boxes tensor");
385 return FALSE;
386 }
387
388 return TRUE;
389 }
390
391 /* Compare c1 and c2
392 * Utility function for sorting candiates based on the scores.
393 */
394 static gint
395 gst_face_detector_tensor_decoder_sort_candidates (gconstpointer c1,
396 gconstpointer c2)
397 {
398 const Candidate *candidate1 = *((Candidate **) c1);
399 const Candidate *candidate2 = *((Candidate **) c2);
400
401 if (*candidate1->score < *candidate2->score) {
402 return 1;
403 } else if (*candidate1->score > *candidate2->score) {
404 return -1;
405 } else {
406 return 0;
407 }
408 }
409
410 static gfloat
411 iou_box (const Candidate * a, const Candidate * b)
412 {
413 gfloat ax1 = a->box[0];
414 gfloat ay1 = a->box[1];
415 gfloat ax2 = a->box[2];
416 gfloat ay2 = a->box[3];
417
418 gfloat bx1 = b->box[0];
419 gfloat by1 = b->box[1];
420 gfloat bx2 = b->box[2];
421 gfloat by2 = b->box[3];
422
423 gfloat xx1 = (ax1 > bx1) ? ax1 : bx1;
424 gfloat yy1 = (ay1 > by1) ? ay1 : by1;
425 gfloat xx2 = (ax2 < bx2) ? ax2 : bx2;
426 gfloat yy2 = (ay2 < by2) ? ay2 : by2;
427
428 gfloat w = xx2 - xx1;
429 gfloat h = yy2 - yy1;
430 if (w < 0.0f || h < 0.0f) {
431 /* No overlap */
432 return 0.0f;
433 }
434
435 /* Area of intersection */
436 gfloat intersection = w * h;
437
438 /* Area of each box */
439 gfloat areaA = (ax2 - ax1) * (ay2 - ay1);
440 gfloat areaB = (bx2 - bx1) * (by2 - by1);
441 if (areaA <= 0.0f || areaB <= 0.0f)
442 return 0.0f;
443
444 /* IoU = intersection / union */
445 gfloat iou = intersection / (areaA + areaB - intersection);
446 return iou;
447 }
448
449 /* hard_nms:
450 * @sel_candidates: array of pointers of selected boxes with scores
451 * @selected: array of pointers of selected boxes with scores after the removal of overlappings
452 * @iou_threshold: threshold for removing boxes based on proportion of the image
453 * @top_k: number of boxes to keep (if top_k <= 0, keep all).
454 * @return: void
455 * Hard NMS:
456 * 1) Keep highest scoring box
457 * 2) Remove boxes with IoU >= iou_threshold
458 * 3) Repeat until no boxes left or we reach top_k
459 */
460 static void
461 hard_nms (const GPtrArray * sel_candidates,
462 GPtrArray * selected, gfloat iou_threshold, gint top_k)
463 {
464 /* Edge case: Handle the case of no input boxes */
465 if (sel_candidates->len == 0) {
466 return;
467 }
468
469 /* We'll mark boxes as "suppressed" using an array of booleans. */
470 gchar *discarded = g_alloca0 (sel_candidates->len); /* 0 => keep, 1 => discard */
471
472 /* The maximum possible output is 'sel_candidates->len'. We'll store the kept boxes into 'selected'. */
473
474 /* Perform NMS. */
475 for (gsize i = 0; i < sel_candidates->len; i++) {
476 if (discarded[i]) {
477 /* Already thrown out due to overlap. */
478 continue;
479 }
480
481 /* Get the current indexed candidate from the selected candidates.
482 * Then store this current box/candidate into final selected candidates array
483 */
484 Candidate *c = (Candidate *) g_ptr_array_index (sel_candidates, i);
485 g_ptr_array_add (selected, c);
486
487 /* If we have reached top_k (and top_k > 0), break. */
488 if (top_k > 0 && selected->len == top_k) {
489 break;
490 }
491
492 /* Suppress any candidate that overlap (IoU >= iou_threshold) with the current one. */
493 for (gsize j = i + 1; j < sel_candidates->len; j++) {
494 if (discarded[j])
495 continue;
496
497 gfloat overlap = iou_box (g_ptr_array_index (sel_candidates, i),
498 g_ptr_array_index (sel_candidates, j));
499 if (overlap >= iou_threshold) {
500 discarded[j] = 1; /* Mark for discard */
501 }
502 }
503 }
504 }
505
506 /* gst_face_detector_tensor_decoder_decode_boxes_f32:
507 * @self: Instance
508 * @boxes_tensor: Buffer containing the boxes tensor
509 * @scores_tensor: Buffer containing the scores/confidences tensor
510 * @rmeta: analytics-meta that is attached to the buffer
511 * @return: void
512 * Decode Face Detection tensors, post-process tensors and store decoded information
513 * into an analytics-meta that is attached to the buffer before been pushed
514 * downstream.
515 */
516 static void
517 gst_face_detector_tensor_decoder_decode_boxes_f32 (GstFaceDetectorTensorDecoder
518 * self, const GstTensor * boxes_tensor, const GstTensor * scores_tensor,
519 GstAnalyticsRelationMeta * rmeta)
520 {
521 GstMapInfo map_info_boxes, map_info_scores;
522 gfloat *candidate, *score;
523 gboolean rv GST_UNUSED_ASSERT;
524 GPtrArray *sel_candidates = self->sel_candidates, *selected = self->selected;
525
526 rv = gst_buffer_map (boxes_tensor->data, &map_info_boxes, GST_MAP_READ);
527 g_assert (rv);
528
529 /* Retrieve memory at index 0 from scores_tensor in READ mode */
530 rv = gst_buffer_map (scores_tensor->data, &map_info_scores, GST_MAP_READ);
531 g_assert (rv);
532
533 GST_LOG_OBJECT (self, "Boxes Tensor shape dims %zu", boxes_tensor->num_dims);
534 GST_LOG_OBJECT (self, "scores Tensor shape dims %zu",
535 scores_tensor->num_dims);
536
537 if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_TRACE) {
538 /* Trace boxes tensor dimensions */
539 for (gsize i = 0; i < boxes_tensor->num_dims; i++) {
540 GST_TRACE_OBJECT (self, "Boxes Tensor dim %zu: %zu", i,
541 boxes_tensor->dims[i]);
542 }
543
544 /* Trace scores tensor dimensions */
545 for (gsize i = 0; i < scores_tensor->num_dims; i++) {
546 GST_TRACE_OBJECT (self, "Scores Tensor dim %zu: %zu", i,
547 scores_tensor->dims[i]);
548 }
549 }
550
551 /* Allocate array to store selected candidates */
552 if (sel_candidates == NULL) {
553 /* Number of candidates can be large, keep the array to avoid frequent
554 * allocation */
555 sel_candidates = g_ptr_array_new_full (boxes_tensor->dims[1], NULL);
556 self->sel_candidates = sel_candidates;
557 selected = g_ptr_array_new_full (boxes_tensor->dims[1], NULL);
558 self->selected = selected;
559 self->candidates = (Candidate *) g_new0 (Candidate, boxes_tensor->dims[1]);
560 } else {
561 /* Reset lengths when we re-use arrays */
562 g_ptr_array_set_size (sel_candidates, 0);
563 g_ptr_array_set_size (selected, 0);
564 }
565
566 score = (gfloat *) map_info_scores.data;
567 candidate = (gfloat *) map_info_boxes.data;
568
569 gsize idx = 0;
570
571 /* For UltraLightFaceDetection:
572 * "boxes" => shape [N,4], where N = 4420
573 * "scores"=> shape [N,2], (background,face)
574 * We'll skip the background (index = 0) and keep the foreground (index = 1).
575 */
576
577 /*
578 * Iterate through the Scores tensor.
579 * Check whether the score exceeds default threshold, if it does, select the score and corresponding box.
580 * Add these selected boxes to the sel_candidates array.
581 * */
582 for (gsize i = 1, j = 0; i < scores_tensor->dims[1] * 2; i += 2, j += 4) {
583 if (score[i] >= self->score_threshold) {
584 self->candidates[idx].index = idx;
585 self->candidates[idx].box = &candidate[j];
586 self->candidates[idx].score = &score[i];
587
588 g_ptr_array_add (sel_candidates, &self->candidates[idx]);
589 idx++;
590 }
591 }
592
593 GST_LOG_OBJECT (self, "Number of selected candidates = %d",
594 sel_candidates->len);
595
596 if (sel_candidates->len == 0) {
597 GST_LOG_OBJECT (self, "No boxes above threshold=%1.2f",
598 self->score_threshold);
599 goto cleanup;
600 }
601
602 if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_TRACE) {
603 for (gsize i = 0; i < sel_candidates->len; i++) {
604 Candidate *c = (Candidate *) g_ptr_array_index (sel_candidates, i);
605 gsize j = 0;
606 for (; j < boxes_tensor->dims[2]; j++) {
607 GST_TRACE_OBJECT (self, "sel_candidates[%zu] = %1.5f ", i + j,
608 c->box[j]);
609 }
610 GST_TRACE_OBJECT (self, "score[%zu] = %1.5f", i + j, c->score[0]);
611 }
612 }
613
614 /*
615 * Sort the sel_candidates array so as to have the candidates in descending order w.r.t. scores
616 */
617 g_ptr_array_sort (sel_candidates,
618 gst_face_detector_tensor_decoder_sort_candidates);
619
620 if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_TRACE) {
621 for (gsize i = 0; i < sel_candidates->len; i++) {
622 Candidate *c = (Candidate *) g_ptr_array_index (sel_candidates, i);
623 GST_TRACE_OBJECT (self, "c[%zu] = %1.5f index = %d", i, c->score[0],
624 c->index);
625 }
626 }
627
628 /* NMS */
629 hard_nms (sel_candidates, selected, self->iou_threshold, -1);
630
631 GST_LOG_OBJECT (self, "Number of faces detected = %d", selected->len);
632 if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_TRACE) {
633 for (gsize i = 0; i < selected->len; i++) {
634 Candidate *c = (Candidate *) g_ptr_array_index (selected, i);
635 GST_TRACE_OBJECT (self,
636 "%zu x1 = %1.5f y1 = %1.5f x2 = %1.5f y2 = %1.5f score = %1.5f",
637 i + 1, c->box[i + 0], c->box[i + 1], c->box[i + 2], c->box[i + 3],
638 c->score[0]);
639 }
640 }
641
642 gsize frame_width = self->video_info.width;
643 gsize frame_height = self->video_info.height;
644
645 /* Convert each final box from normalized to pixel coords and attach to meta. */
646 for (gint i = 0; i < selected->len; i++) {
647 Candidate *c = (Candidate *) g_ptr_array_index (selected, i);
648 gfloat x1 = c->box[0] * frame_width;
649 gfloat y1 = c->box[1] * frame_height;
650 gfloat x2 = c->box[2] * frame_width;
651 gfloat y2 = c->box[3] * frame_height;
652 gfloat w_ = x2 - x1;
653 gfloat h_ = y2 - y1;
654
655 /* Add to analytics meta: (x, y, width, height). */
656 gst_analytics_relation_meta_add_od_mtd (rmeta, FACE_QUARK,
657 (gint) (x1 + 0.5f), (gint) (y1 + 0.5f),
658 (gint) (w_ + 0.5f), (gint) (h_ + 0.5f), c->score[0], NULL);
659 }
660
661 cleanup:
662
663 /* Unmap */
664 gst_buffer_unmap (boxes_tensor->data, &map_info_boxes);
665 gst_buffer_unmap (scores_tensor->data, &map_info_scores);
666 }
667
668 /* gst_face_detector_tensor_decoder_transform_ip:
669 * @trans: Instance
670 * @buf:inout: Buffer containing media and where tensors can be attached
671 * @return: Flow errors
672 * Decode Face Detection tensors, post-process tensors and store decoded information
673 * into an analytics-meta that is attached to the buffer before been pushed
674 * downstream.
675 */
676 static GstFlowReturn
677 gst_face_detector_tensor_decoder_transform_ip (GstBaseTransform * trans,
678 GstBuffer * buf)
679 {
680 GstFaceDetectorTensorDecoder *self = GST_FACE_DETECTOR_TENSOR_DECODER (trans);
681 const GstTensor *boxes_tensor, *scores_tensor;
682 GstAnalyticsRelationMeta *rmeta;
683
684 /* Retrive the desired Face Detection tensors.
685 * Return Flow Error if the desired tensors were not supported. */
686 if (!gst_face_detector_tensor_decoder_get_tensor_meta (self, buf,
687 &boxes_tensor, &scores_tensor)) {
688 GST_ELEMENT_ERROR (self, STREAM, DECODE, (NULL),
689 ("Tensor doesn't have the expected data type or shape."));
690 return GST_FLOW_ERROR;
691 }
692
693 rmeta = gst_buffer_add_analytics_relation_meta (buf);
694 g_assert (rmeta != NULL);
695
696 /* Decode boxes_tensor, scores_tensor and attach the information in a structured way
697 * to rmeta. */
698 gst_face_detector_tensor_decoder_decode_boxes_f32 (self, boxes_tensor,
699 scores_tensor, rmeta);
700
701 return GST_FLOW_OK;
702 }