Debian Codegraph @

gst/tensordecoders/gstssdtensordec.c

raw
1 /*
2 * GStreamer gstreamer-ssdtensordec
3 * Copyright (C) 2021,2025 Collabora Ltd.
4 *
5 * gstssdtensordec.c
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
16 *
17 * You should have received a copy of the GNU Library General Public
18 * License along with this library; if not, write to the
19 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
20 * Boston, MA 02110-1301, USA.
21 */
22
23 /**
24 * SECTION:element-ssdtensordec
25 * @short_description: Decode tensors from inference using a SSD neural network
26 *
27 * This element can parse per-buffer inference tensor meta data generated by an upstream
28 * inference element
29 *
30 *
31 * ## Example launch command:
32 *
33 * Test image file, model file (SSD) and label file can be found here :
34 * https://gitlab.collabora.com/gstreamer/onnx-models
35 *
36 * GST_DEBUG=ssdtensordec:5 \
37 * gst-launch-1.0 multifilesrc location=onnx-models/images/bus.jpg ! \
38 * jpegdec ! videoconvert ! onnxinference execution-provider=cpu model-file=onnx-models/models/ssd_mobilenet_v1_coco.onnx ! \
39 * ssdtensordec label-file=onnx-models/labels/COCO_classes.txt ! videoconvert ! autovideosink
40 *
41 * Since: 1.28
42 */
43
44 /**
45 * SECTION:element-ssdobjectdetector
46 * @short_description: Detect objects in video buffers using SSD neural network
47 *
48 * This element can parse per-buffer inference tensor meta data generated by an upstream
49 * inference element
50 *
51 *
52 * ## Example launch command:
53 *
54 * Test image file, model file (SSD) and label file can be found here :
55 * https://gitlab.collabora.com/gstreamer/onnx-models
56 *
57 * GST_DEBUG=ssdtensordec:5 \
58 * gst-launch-1.0 multifilesrc location=onnx-models/images/bus.jpg ! \
59 * jpegdec ! videoconvert ! onnxinference execution-provider=cpu model-file=onnx-models/models/ssd_mobilenet_v1_coco.onnx ! \
60 * ssdtensordec label-file=onnx-models/labels/COCO_classes.txt ! videoconvert ! autovideosink
61 *
62 * Since: 1.20
63 * Deprecated: 1.28
64 */
65 #ifdef HAVE_CONFIG_H
66 #include "config.h"
67 #endif
68
69 #include "gstssdtensordec.h"
70
71 #include <gio/gio.h>
72
73 #include <gst/gst.h>
74 #include <gst/video/video.h>
75 #include <gst/analytics/analytics.h>
76
77 /* Object detection tensor id strings */
78 #define GROUP_ID "ssd-mobilenet-v1-variant-1-out"
79 #define GST_MODEL_OBJECT_DETECTOR_BOXES "ssd-mobilenet-v1-variant-1-out-boxes"
80 #define GST_MODEL_OBJECT_DETECTOR_SCORES "ssd-mobilenet-v1-variant-1-out-scores"
81 #define GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS "generic-variant-1-out-count"
82 #define GST_MODEL_OBJECT_DETECTOR_CLASSES "ssd-mobilenet-v1-variant-1-out-classes"
83
84 GST_DEBUG_CATEGORY_STATIC (ssd_tensor_dec_debug);
85 #define GST_CAT_DEFAULT ssd_tensor_dec_debug
86 GST_ELEMENT_REGISTER_DEFINE (ssd_tensor_dec, "ssdtensordec",
87 GST_RANK_SECONDARY, GST_TYPE_SSD_TENSOR_DEC);
88
89 GST_DEBUG_CATEGORY_STATIC (ssd_object_detector_debug);
90 GST_ELEMENT_REGISTER_DEFINE (ssd_object_detector, "ssdobjectdetector",
91 GST_RANK_NONE, GST_TYPE_SSD_OBJECT_DETECTOR);
92 /* GstSsdTensorDec properties */
93 enum
94 {
95 PROP_0,
96 PROP_LABEL_FILE,
97 PROP_SCORE_THRESHOLD,
98 PROP_SIZE_THRESHOLD
99 };
100
101 #define GST_SSD_TENSOR_DEC_DEFAULT_SCORE_THRESHOLD 0.3f /* 0 to 1 */
102 #define GST_SSD_TENSOR_DEC_DEFAULT_SIZE_THRESHOLD 0.9f /* 0 to 1 */
103
104 static GstStaticPadTemplate gst_ssd_tensor_dec_src_template =
105 GST_STATIC_PAD_TEMPLATE ("src",
106 GST_PAD_SRC,
107 GST_PAD_ALWAYS,
108 GST_STATIC_CAPS ("video/x-raw")
109 );
110
111 /* *INDENT-OFF* */
112
113 static GstStaticPadTemplate gst_ssd_tensor_dec_sink_template =
114 GST_STATIC_PAD_TEMPLATE ("sink",
115 GST_PAD_SINK,
116 GST_PAD_ALWAYS,
117 GST_STATIC_CAPS ("video/x-raw,"
118 "tensors=(structure)["
119 "tensorgroups,"
120 GROUP_ID"=(/uniquelist){"
121 "(GstCaps)["
122 "tensor/strided,"
123 "tensor-id="GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS","
124 "dims=(int)<[0, max]>,"
125 "dims-order=(string)row-major,"
126 "type=(string)float32;],"
127 "(GstCaps)["
128 "tensor/strided,"
129 "tensor-id="GST_MODEL_OBJECT_DETECTOR_SCORES","
130 "dims=(int)<[0, max],0>,"
131 "dims-order=(string)row-major,"
132 "type=(string)float32;],"
133 "(GstCaps)["
134 "tensor/strided,"
135 "tensor-id="GST_MODEL_OBJECT_DETECTOR_BOXES","
136 "dims=(int)<[0, max],0,4>,"
137 "dims-order=(string)row-major,"
138 "type=(string)float32;],"
139 "(GstCaps)["
140 "tensor/strided,"
141 "tensor-id="GST_MODEL_OBJECT_DETECTOR_CLASSES","
142 "dims=(int)<[0, max],0>,"
143 "dims-order=(string)row-major,"
144 "type=(string)float32;]"
145 "}"
146 "];"
147 ));
148
149 /* *INDENT-ON* */
150
151 static void gst_ssd_tensor_dec_set_property (GObject * object,
152 guint prop_id, const GValue * value, GParamSpec * pspec);
153 static void gst_ssd_tensor_dec_get_property (GObject * object,
154 guint prop_id, GValue * value, GParamSpec * pspec);
155 static void gst_ssd_tensor_dec_finalize (GObject * object);
156 static GstFlowReturn gst_ssd_tensor_dec_transform_ip (GstBaseTransform *
157 trans, GstBuffer * buf);
158 static gboolean gst_ssd_tensor_dec_process (GstBaseTransform * trans,
159 GstBuffer * buf);
160 static gboolean
161 gst_ssd_tensor_dec_set_caps (GstBaseTransform * trans, GstCaps * incaps,
162 GstCaps * outcaps);
163
164 G_DEFINE_TYPE (GstSsdTensorDec, gst_ssd_tensor_dec, GST_TYPE_BASE_TRANSFORM);
165
166 static void
167 gst_ssd_tensor_dec_class_init (GstSsdTensorDecClass * klass)
168 {
169 GObjectClass *gobject_class = (GObjectClass *) klass;
170 GstElementClass *element_class = (GstElementClass *) klass;
171 GstBaseTransformClass *basetransform_class = (GstBaseTransformClass *) klass;
172
173 GST_DEBUG_CATEGORY_INIT (ssd_tensor_dec_debug, "ssdtensordec",
174 0, "ssdtensordec");
175 gobject_class->set_property = gst_ssd_tensor_dec_set_property;
176 gobject_class->get_property = gst_ssd_tensor_dec_get_property;
177 gobject_class->finalize = gst_ssd_tensor_dec_finalize;
178
179 /**
180 * GstSsdTensorDec:label-file
181 *
182 * Label file
183 *
184 * Since: 1.24
185 */
186 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_LABEL_FILE,
187 g_param_spec_string ("label-file",
188 "Label file", "Label file", NULL, (GParamFlags)
189 (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
190
191 /**
192 * GstSsdTensorDec:score-threshold
193 *
194 * Threshold for deciding when to remove boxes based on score
195 *
196 * Since: 1.24
197 */
198 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_SCORE_THRESHOLD,
199 g_param_spec_float ("score-threshold",
200 "Score threshold",
201 "Threshold for deciding when to remove boxes based on score",
202 0.0, 1.0, GST_SSD_TENSOR_DEC_DEFAULT_SCORE_THRESHOLD, (GParamFlags)
203 (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
204
205 /**
206 * GstSsdTensorDec:size-threshold
207 *
208 * Threshold for deciding when to remove boxes based on proportion of the image
209 *
210 * Since: 1.26
211 */
212 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_SIZE_THRESHOLD,
213 g_param_spec_float ("size-threshold",
214 "Size threshold",
215 "Threshold for deciding when to remove boxes based on proportion of the image",
216 0.0, 1.0, GST_SSD_TENSOR_DEC_DEFAULT_SIZE_THRESHOLD, (GParamFlags)
217 (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
218
219 gst_element_class_set_static_metadata (element_class,
220 "SSD MobileNet Object Detector tensor decoder",
221 "Tensordecoder/Video",
222 "Apply tensor output from inference to detect objects in video frames",
223 "Aaron Boxer <[email protected]>, Marcus Edel <[email protected]>");
224
225 gst_element_class_add_pad_template (element_class,
226 gst_static_pad_template_get (&gst_ssd_tensor_dec_sink_template));
227
228 gst_element_class_add_pad_template (element_class,
229 gst_static_pad_template_get (&gst_ssd_tensor_dec_src_template));
230 basetransform_class->transform_ip =
231 GST_DEBUG_FUNCPTR (gst_ssd_tensor_dec_transform_ip);
232 basetransform_class->set_caps =
233 GST_DEBUG_FUNCPTR (gst_ssd_tensor_dec_set_caps);
234 }
235
236 static void
237 gst_ssd_tensor_dec_init (GstSsdTensorDec * self)
238 {
239 self->size_threshold = GST_SSD_TENSOR_DEC_DEFAULT_SIZE_THRESHOLD;
240 self->score_threshold = GST_SSD_TENSOR_DEC_DEFAULT_SCORE_THRESHOLD;
241 GST_PAD_UNSET_ACCEPT_INTERSECT (self->basetransform.sinkpad);
242 }
243
244 static void
245 gst_ssd_tensor_dec_finalize (GObject * object)
246 {
247 GstSsdTensorDec *self = GST_SSD_TENSOR_DEC (object);
248
249 g_free (self->label_file);
250 g_clear_pointer (&self->labels, g_array_unref);
251
252 G_OBJECT_CLASS (gst_ssd_tensor_dec_parent_class)->finalize (object);
253 }
254
255 static GArray *
256 read_labels (const char *labels_file)
257 {
258 GArray *array;
259 GFile *file = g_file_new_for_path (labels_file);
260 GFileInputStream *file_stream;
261 GDataInputStream *data_stream;
262 GError *error = NULL;
263 gchar *line;
264
265 file_stream = g_file_read (file, NULL, &error);
266 g_object_unref (file);
267 if (!file_stream) {
268 GST_WARNING ("Could not open file %s: %s\n", labels_file, error->message);
269 g_clear_error (&error);
270 return NULL;
271 }
272
273 data_stream = g_data_input_stream_new (G_INPUT_STREAM (file_stream));
274 g_object_unref (file_stream);
275
276 array = g_array_new (FALSE, FALSE, sizeof (GQuark));
277
278 while ((line = g_data_input_stream_read_line (data_stream, NULL, NULL,
279 &error))) {
280 GQuark label = g_quark_from_string (line);
281 g_array_append_val (array, label);
282 g_free (line);
283 }
284
285 g_object_unref (data_stream);
286
287 if (error) {
288 GST_WARNING ("Could not open file %s: %s", labels_file, error->message);
289 g_array_free (array, TRUE);
290 g_clear_error (&error);
291 return NULL;
292 }
293
294 if (array->len == 0) {
295 g_array_free (array, TRUE);
296 return NULL;
297 }
298
299 return array;
300 }
301
302 static void
303 gst_ssd_tensor_dec_set_property (GObject * object, guint prop_id,
304 const GValue * value, GParamSpec * pspec)
305 {
306 GstSsdTensorDec *self = GST_SSD_TENSOR_DEC (object);
307 const gchar *filename;
308
309 switch (prop_id) {
310 case PROP_LABEL_FILE:
311 {
312 GArray *labels;
313
314 filename = g_value_get_string (value);
315 labels = read_labels (filename);
316
317 if (labels) {
318 g_free (self->label_file);
319 self->label_file = g_strdup (filename);
320 g_clear_pointer (&self->labels, g_array_unref);
321 self->labels = labels;
322 } else {
323 GST_WARNING_OBJECT (self, "Label file '%s' not found!", filename);
324 }
325 }
326 break;
327 case PROP_SCORE_THRESHOLD:
328 GST_OBJECT_LOCK (self);
329 self->score_threshold = g_value_get_float (value);
330 GST_OBJECT_UNLOCK (self);
331 break;
332 case PROP_SIZE_THRESHOLD:
333 GST_OBJECT_LOCK (self);
334 self->size_threshold = g_value_get_float (value);
335 GST_OBJECT_UNLOCK (self);
336 break;
337 default:
338 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
339 break;
340 }
341 }
342
343 static void
344 gst_ssd_tensor_dec_get_property (GObject * object, guint prop_id,
345 GValue * value, GParamSpec * pspec)
346 {
347 GstSsdTensorDec *self = GST_SSD_TENSOR_DEC (object);
348
349 switch (prop_id) {
350 case PROP_LABEL_FILE:
351 g_value_set_string (value, self->label_file);
352 break;
353 case PROP_SCORE_THRESHOLD:
354 GST_OBJECT_LOCK (self);
355 g_value_set_float (value, self->score_threshold);
356 GST_OBJECT_UNLOCK (self);
357 break;
358 case PROP_SIZE_THRESHOLD:
359 GST_OBJECT_LOCK (self);
360 g_value_set_float (value, self->size_threshold);
361 GST_OBJECT_UNLOCK (self);
362 break;
363 default:
364 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
365 break;
366 }
367 }
368
369 static gboolean
370 gst_ssd_tensor_dec_get_tensors (GstSsdTensorDec * object_detector,
371 GstBuffer * buf, const GstTensor ** classes_tensor,
372 const GstTensor ** numdetect_tensor, const GstTensor ** scores_tensor,
373 const GstTensor ** boxes_tensor)
374 {
375 GstMeta *meta = NULL;
376 gpointer iter_state = NULL;
377 static const gsize BOXES_DIMS[] = { 1, G_MAXSIZE, 4 };
378 static const gsize NUM_DETECT_DIMS[] = { 1 };
379 static const gsize SCORES_CLASSES_DIMS[] = { 1, G_MAXSIZE };
380
381 if (!gst_buffer_get_meta (buf, GST_TENSOR_META_API_TYPE)) {
382 GST_DEBUG_OBJECT (object_detector,
383 "missing tensor meta from buffer %" GST_PTR_FORMAT, buf);
384 return FALSE;
385 }
386
387 // find object detector meta
388
389 while ((meta = gst_buffer_iterate_meta_filtered (buf, &iter_state,
390 GST_TENSOR_META_API_TYPE))) {
391 GstTensorMeta *tmeta = (GstTensorMeta *) meta;
392
393 *boxes_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
394 g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_BOXES),
395 GST_TENSOR_DATA_TYPE_FLOAT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 3,
396 BOXES_DIMS);
397 if (*boxes_tensor == NULL)
398 *boxes_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
399 g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_BOXES),
400 GST_TENSOR_DATA_TYPE_UINT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 3,
401 BOXES_DIMS);
402 if (*boxes_tensor == NULL)
403 continue;
404
405 *scores_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
406 g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_SCORES),
407 GST_TENSOR_DATA_TYPE_FLOAT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 2,
408 SCORES_CLASSES_DIMS);
409 if (*scores_tensor == NULL)
410 *scores_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
411 g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_SCORES),
412 GST_TENSOR_DATA_TYPE_UINT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 2,
413 SCORES_CLASSES_DIMS);
414 if (*scores_tensor == NULL)
415 continue;
416
417 *numdetect_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
418 g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS),
419 GST_TENSOR_DATA_TYPE_FLOAT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 1,
420 NUM_DETECT_DIMS);
421 if (*numdetect_tensor == NULL)
422 *numdetect_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
423 g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS),
424 GST_TENSOR_DATA_TYPE_UINT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 1,
425 NUM_DETECT_DIMS);
426 if (*numdetect_tensor == NULL)
427 continue;
428
429 *classes_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
430 g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_CLASSES),
431 GST_TENSOR_DATA_TYPE_FLOAT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 2,
432 SCORES_CLASSES_DIMS);
433 if (*classes_tensor == NULL)
434 *classes_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
435 g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_CLASSES),
436 GST_TENSOR_DATA_TYPE_UINT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 2,
437 SCORES_CLASSES_DIMS);
438
439 return TRUE;
440 }
441
442 return FALSE;
443 }
444
445 static gboolean
446 gst_ssd_tensor_dec_set_caps (GstBaseTransform * trans, GstCaps * incaps,
447 GstCaps * outcaps)
448 {
449 GstSsdTensorDec *self = GST_SSD_TENSOR_DEC (trans);
450
451 if (!gst_video_info_from_caps (&self->video_info, incaps)) {
452 GST_ERROR_OBJECT (self, "Failed to parse caps");
453 return FALSE;
454 }
455
456 return TRUE;
457 }
458
459 static GstFlowReturn
460 gst_ssd_tensor_dec_transform_ip (GstBaseTransform * trans, GstBuffer * buf)
461 {
462 if (!gst_base_transform_is_passthrough (trans)) {
463 if (!gst_ssd_tensor_dec_process (trans, buf)) {
464 GST_ELEMENT_ERROR (trans, STREAM, FAILED,
465 (NULL), ("ssd object detection failed"));
466 return GST_FLOW_ERROR;
467 }
468 }
469
470 return GST_FLOW_OK;
471 }
472
473 #define DEFINE_GET_FUNC(TYPE, MAX) \
474 static gboolean \
475 get_ ## TYPE ## _at_index (const GstTensor *tensor, GstMapInfo *map, \
476 guint index, TYPE * out) \
477 { \
478 switch (tensor->data_type) { \
479 case GST_TENSOR_DATA_TYPE_FLOAT32: { \
480 float *f = (float *) map->data; \
481 if (sizeof(*f) * (index + 1) > map->size) \
482 return FALSE; \
483 *out = f[index]; \
484 break; \
485 } \
486 case GST_TENSOR_DATA_TYPE_UINT32: { \
487 guint32 *u = (guint32 *) map->data; \
488 if (sizeof(*u) * (index + 1) > map->size) \
489 return FALSE; \
490 *out = u[index]; \
491 break; \
492 } \
493 default: \
494 GST_ERROR ("Only float32 and int32 tensors are understood"); \
495 return FALSE; \
496 } \
497 return TRUE; \
498 }
499
500 DEFINE_GET_FUNC (guint32, UINT32_MAX);
501 DEFINE_GET_FUNC (float, FLOAT_MAX);
502 #undef DEFINE_GET_FUNC
503
504 static void
505 extract_bounding_boxes (GstSsdTensorDec * self, gsize w, gsize h,
506 GstAnalyticsRelationMeta * rmeta, const GstTensor * classes_tensor,
507 const GstTensor * numdetect_tensor, const GstTensor * scores_tensor,
508 const GstTensor * boxes_tensor)
509 {
510 GstMapInfo boxes_map = GST_MAP_INFO_INIT;
511 GstMapInfo numdetect_map = GST_MAP_INFO_INIT;
512 GstMapInfo scores_map = GST_MAP_INFO_INIT;
513 GstMapInfo classes_map = GST_MAP_INFO_INIT;
514
515 guint num_detections = 0;
516
517 if (numdetect_tensor == NULL || scores_tensor == NULL || boxes_tensor == NULL) {
518 GST_WARNING ("Missing tensor data expected for SSD model");
519 return;
520 }
521
522 if (!gst_buffer_map (numdetect_tensor->data, &numdetect_map, GST_MAP_READ)) {
523 GST_ERROR_OBJECT (self, "Failed to map numdetect tensor memory");
524 goto cleanup;
525 }
526
527 if (!gst_buffer_map (boxes_tensor->data, &boxes_map, GST_MAP_READ)) {
528 GST_ERROR_OBJECT (self, "Failed to map boxes tensor memory");
529 goto cleanup;
530 }
531
532 if (!gst_buffer_map (scores_tensor->data, &scores_map, GST_MAP_READ)) {
533 GST_ERROR_OBJECT (self, "Failed to map scores tensor memory");
534 goto cleanup;
535 }
536
537 if (classes_tensor &&
538 !gst_buffer_map (classes_tensor->data, &classes_map, GST_MAP_READ)) {
539 GST_DEBUG_OBJECT (self, "Failed to map classes tensor memory");
540 goto cleanup;
541 }
542
543
544 if (!get_guint32_at_index (numdetect_tensor, &numdetect_map,
545 0, &num_detections)) {
546 GST_ERROR_OBJECT (self, "Failed to get the number of detections");
547 goto cleanup;
548 }
549
550
551 GST_LOG_OBJECT (self, "Model claims %u detections", num_detections);
552 num_detections = MIN (num_detections, scores_tensor->dims[1]);
553 num_detections = MIN (num_detections, boxes_tensor->dims[1]);
554 if (classes_tensor)
555 num_detections = MIN (num_detections, classes_tensor->dims[1]);
556 GST_LOG_OBJECT (self, "Model really has %u detections"
557 " (%zu scores, %zu boxes, %zu classes)", num_detections,
558 scores_tensor->dims[1], boxes_tensor->dims[1],
559 classes_tensor ? classes_tensor->dims[1] : 0);
560
561 for (int i = 0; i < num_detections; i++) {
562 float score;
563 float x, y, bwidth, bheight;
564 gint x_i, y_i, bwidth_i, bheight_i;
565 guint32 bclass;
566 GQuark label = 0;
567 GstAnalyticsODMtd odmtd;
568
569 if (!get_float_at_index (scores_tensor, &scores_map, i, &score))
570 continue;
571
572 GST_LOG_OBJECT (self, "Detection %u score is %f", i, score);
573 if (score < self->score_threshold)
574 continue;
575
576 if (!get_float_at_index (boxes_tensor, &boxes_map, i * 4, &y))
577 continue;
578 if (!get_float_at_index (boxes_tensor, &boxes_map, i * 4 + 1, &x))
579 continue;
580 if (!get_float_at_index (boxes_tensor, &boxes_map, i * 4 + 2, &bheight))
581 continue;
582 if (!get_float_at_index (boxes_tensor, &boxes_map, i * 4 + 3, &bwidth))
583 continue;
584
585 if (CLAMP (bwidth, 0, 1) * CLAMP (bheight, 0, 1) > self->size_threshold) {
586 GST_LOG_OBJECT (self, "Object at (%fx%f)=%f > %f, skipping",
587 CLAMP (bwidth, 0, 1), CLAMP (bheight, 0, 1),
588 CLAMP (bwidth, 0, 1) * CLAMP (bheight, 0, 1), self->size_threshold);
589 continue;
590 }
591
592 if (self->labels && classes_map.memory &&
593 get_guint32_at_index (classes_tensor, &classes_map, i, &bclass)) {
594 if (bclass < self->labels->len)
595 label = g_array_index (self->labels, GQuark, bclass);
596 }
597
598 x_i = x * w;
599 y_i = y * h;
600 bheight_i = (bheight * h) - y_i;
601 bwidth_i = (bwidth * w) - x_i;
602
603 if (gst_analytics_relation_meta_add_od_mtd (rmeta, label,
604 x_i, y_i, bwidth_i, bheight_i, score, &odmtd))
605 GST_DEBUG_OBJECT (self,
606 "Object detected with label[%u] : %s, score: %f, bound box: %dx%d at (%d,%d)",
607 bclass, g_quark_to_string (label), score, bwidth_i, bheight_i, x_i,
608 y_i);
609 else
610 GST_WARNING_OBJECT (self, "Could not add detection to meta");
611 }
612
613 cleanup:
614
615 if (numdetect_map.memory)
616 gst_buffer_unmap (numdetect_tensor->data, &numdetect_map);
617 if (classes_map.memory)
618 gst_buffer_unmap (classes_tensor->data, &classes_map);
619 if (scores_map.memory)
620 gst_buffer_unmap (scores_tensor->data, &scores_map);
621 if (boxes_map.memory)
622 gst_buffer_unmap (boxes_tensor->data, &boxes_map);
623 }
624
625
626 static gboolean
627 gst_ssd_tensor_dec_process (GstBaseTransform * trans, GstBuffer * buf)
628 {
629 GstSsdTensorDec *self = GST_SSD_TENSOR_DEC (trans);
630 GstAnalyticsRelationMeta *rmeta;
631 const GstTensor *classes_tensor = NULL;
632 const GstTensor *numdetect_tensor = NULL;
633 const GstTensor *scores_tensor = NULL;
634 const GstTensor *boxes_tensor = NULL;
635
636 // get all tensor metas
637 if (!gst_ssd_tensor_dec_get_tensors (self, buf,
638 &classes_tensor, &numdetect_tensor, &scores_tensor, &boxes_tensor)) {
639 GST_WARNING_OBJECT (trans, "missing tensor meta");
640 return TRUE;
641 } else {
642 rmeta = gst_buffer_add_analytics_relation_meta (buf);
643 g_assert (rmeta);
644 }
645
646 extract_bounding_boxes (self, self->video_info.width,
647 self->video_info.height, rmeta, classes_tensor, numdetect_tensor,
648 scores_tensor, boxes_tensor);
649
650 return TRUE;
651 }
652
653 G_DEFINE_TYPE (GstSsdObjectDetector, gst_ssd_object_detector,
654 GST_TYPE_SSD_TENSOR_DEC);
655
656 static void
657 gst_ssd_object_detector_class_init (GstSsdObjectDetectorClass * klass)
658 {
659 GST_DEBUG_CATEGORY_INIT (ssd_object_detector_debug, "ssdobjectdetector",
660 0, "ssd object detector category");
661
662 gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass),
663 "SSD MobileNet Object Detector tensor decoder (Deprecated)",
664 "Tensordecoder/Video",
665 "Apply tensor output from inference to detect objects in video frames",
666 "Aaron Boxer <[email protected]>, Marcus Edel <[email protected]>");
667
668 gst_type_mark_as_plugin_api (GST_TYPE_SSD_TENSOR_DEC, 0);
669 }
670
671 static void
672 gst_ssd_object_detector_init (GstSsdObjectDetector * self)
673 {
674 GST_CAT_WARNING (ssd_object_detector_debug, "ssdobjectdetector is "
675 "deprecated, use ssdtensordec instead");
676 }