Debian Codegraph @

gst/tensordecoders/gsttensordecodebin.c

raw
1 /* GStreamer object detection overlay
2 * Copyright (C) <2024, 2025> Collabora Ltd.
3 * @author: Daniel Morin <[email protected]>
4 *
5 * gsttensordecodebin.c
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
16 *
17 * You should have received a copy of the GNU Library General Public
18 * License along with this library; if not, write to the
19 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
20 * Boston, MA 02110-1301, USA.
21 */
22
23 /**
24 * SECTION:element-tensordecodebin
25 * @short_description: Find and instantiate compatible tensor decoder
26 *
27 * This element instantiate a tensor decoder compatible with upstream caps.
28 *
29 * ## Example launch command:
30 * |[
31 * gst-launch-1.0 filesrc location=/onnx-models/images/bus.jpg !
32 * ! jpegdec ! videoconvert ! onnxinference execution-provider=cpu
33 * model-file=/onnx-models/models/ssd_mobilenet_v1_coco.onnx
34 * ! tensordecodebin ! objectdetectionoverlay ! videoconvert ! imagefreeze
35 * ! autovideosink
36 * ]| Assuming the model is a object detection model this pipeline will instantiate
37 * a tensor decoder compatible upstream tensor caps.
38 *
39 * Since: 1.28
40 */
41
42 #include <gst/gst.h>
43 #ifdef HAVE_CONFI_H
44 #include "config.h"
45 #endif
46
47 #include "gsttensordecodebin.h"
48
49 #include <gst/gst.h>
50
51
52 GST_DEBUG_CATEGORY_STATIC (tensordecodebin_debug);
53 #define GST_CAT_DEFAULT tensordecodebin_debug
54
55 GST_ELEMENT_REGISTER_DEFINE (tensordecodebin, "tensordecodebin",
56 GST_RANK_NONE, GST_TYPE_TENSORDECODEBIN);
57
58 static GstStaticPadTemplate gst_tensordecodebin_sink_template =
59 GST_STATIC_PAD_TEMPLATE ("sink",
60 GST_PAD_SINK,
61 GST_PAD_ALWAYS,
62 GST_STATIC_CAPS_ANY);
63
64 static GstStaticPadTemplate gst_tensordecodebin_src_template =
65 GST_STATIC_PAD_TEMPLATE ("src",
66 GST_PAD_SRC,
67 GST_PAD_ALWAYS,
68 GST_STATIC_CAPS_ANY);
69
70 static gboolean
71 gst_tensordecodebin_sink_query (GstPad * pad, GstObject * parend, GstQuery *
72 query);
73
74 static gboolean
75 gst_tensordecodebin_sink_event (GstPad * pad, GstObject * parent, GstEvent *
76 event);
77
78 static void gst_tensordecodebin_finalize (GObject * object);
79
80 G_DEFINE_TYPE (GstTensorDecodeBin, gst_tensordecodebin, GST_TYPE_BIN);
81
82 static void
83 gst_tensordecodebin_class_init (GstTensorDecodeBinClass * klass)
84 {
85 GObjectClass *gobject_class = (GObjectClass *) klass;
86 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
87
88 GST_DEBUG_CATEGORY_INIT (tensordecodebin_debug, "tensordecodebin", 0,
89 "Tensor decode bin");
90
91 /* Element description. */
92 gst_element_class_set_static_metadata (element_class, "tensordecodebin",
93 "Tensor Decoder Bin",
94 "Tensor Decode Bin", "Daniel Morin <[email protected]>");
95
96 gst_element_class_add_pad_template (element_class,
97 gst_static_pad_template_get (&gst_tensordecodebin_src_template));
98
99 gst_element_class_add_pad_template (element_class,
100 gst_static_pad_template_get (&gst_tensordecodebin_sink_template));
101
102 gobject_class->finalize = gst_tensordecodebin_finalize;
103 }
104
105 static void
106 gst_tensordecodebin_init (GstTensorDecodeBin * self)
107 {
108 GstPadTemplate *pad_tmpl;
109 pad_tmpl = gst_static_pad_template_get (&gst_tensordecodebin_sink_template);
110 self->sinkpad = gst_ghost_pad_new_no_target_from_template ("sink", pad_tmpl);
111
112 gst_clear_object (&pad_tmpl);
113 pad_tmpl = gst_static_pad_template_get (&gst_tensordecodebin_src_template);
114 self->srcpad = gst_ghost_pad_new_no_target_from_template ("src", pad_tmpl);
115 gst_clear_object (&pad_tmpl);
116
117 self->last_event_caps = NULL;
118 self->factories_cookie = 0;
119 self->aggregated_caps = NULL;
120
121 gst_pad_set_query_function (self->sinkpad, gst_tensordecodebin_sink_query);
122 gst_pad_set_event_function_full (self->sinkpad,
123 gst_tensordecodebin_sink_event, self, NULL);
124
125 gst_element_add_pad (GST_ELEMENT (self), self->sinkpad);
126 gst_element_add_pad (GST_ELEMENT (self), self->srcpad);
127 }
128
129 static void
130 gst_tensordecodebin_finalize (GObject * object)
131 {
132 GstTensorDecodeBin *self = GST_TENSORDECODEBIN (object);
133 g_list_free_full (self->tensordec_factories, gst_object_unref);
134 gst_clear_caps (&self->last_event_caps);
135 gst_clear_caps (&self->aggregated_caps);
136 G_OBJECT_CLASS (gst_tensordecodebin_parent_class)->finalize (object);
137 }
138
139 static gboolean
140 decoder_filter (GstPluginFeature * feature, GstTensorDecodeBin * self)
141 {
142 const gchar *klass;
143 GstElementFactory *fact;
144 GstElementClass *self_class = GST_ELEMENT_GET_CLASS (self);
145
146 /* we only care about element factories */
147 if (G_UNLIKELY (!GST_IS_ELEMENT_FACTORY (feature)))
148 return FALSE;
149
150 fact = GST_ELEMENT_FACTORY_CAST (feature);
151
152 klass = gst_element_factory_get_metadata (fact, GST_ELEMENT_METADATA_KLASS);
153
154 /* Filter on Tensordecoder Klass */
155 if (strstr (klass, "Tensordecoder")) {
156
157 /* Skip ourself */
158 if (fact == self_class->elementfactory)
159 return FALSE;
160
161 /* Only keep element with rank equal or above marginal */
162 if (gst_plugin_feature_get_rank (feature) < GST_RANK_MARGINAL)
163 return FALSE;
164
165 GST_DEBUG_OBJECT (self, "adding %s factory", GST_OBJECT_NAME (fact));
166 return TRUE;
167 }
168
169 return FALSE;
170 }
171
172 static GList *
173 gst_tensordecodebin_get_or_load_tensordec_factories_unlocked (GstTensorDecodeBin
174 * self)
175 {
176 guint32 cookie;
177 GList *all_tensordec_factories;
178
179 cookie = gst_registry_get_feature_list_cookie (gst_registry_get ());
180 if (!self->tensordec_factories || self->factories_cookie != cookie) {
181
182 if (self->tensordec_factories)
183 g_list_free_full (self->tensordec_factories, gst_object_unref);
184
185 all_tensordec_factories =
186 g_list_sort (gst_registry_feature_filter (gst_registry_get (),
187 (GstPluginFeatureFilter) decoder_filter, FALSE, self),
188 (GCompareFunc) gst_plugin_feature_rank_compare_func);
189
190 self->tensordec_factories = all_tensordec_factories;
191 self->factories_cookie = cookie;
192 gst_clear_caps (&self->aggregated_caps);
193 }
194
195 return g_list_copy_deep (self->tensordec_factories,
196 (GCopyFunc) gst_object_ref, NULL);
197 }
198
199 static GList *
200 gst_tensordecodebin_get_or_load_tensordec_factories (GstTensorDecodeBin * self)
201 {
202 GList *factories;
203 GST_OBJECT_LOCK (self);
204 factories =
205 gst_tensordecodebin_get_or_load_tensordec_factories_unlocked (self);
206 GST_OBJECT_UNLOCK (self);
207 return factories;
208 }
209
210 static void
211 _remove_all_elements (GstBin * bin)
212 {
213 GstElement *e;
214 GList *childs, *l;
215
216 GST_DEBUG_OBJECT (bin, "Removing all childs");
217
218 GST_OBJECT_LOCK (bin);
219 childs = g_list_copy_deep (bin->children, (GCopyFunc) gst_object_ref, NULL);
220 GST_OBJECT_UNLOCK (bin);
221
222 l = childs;
223 while (l) {
224 GST_TRACE_OBJECT (bin, "Removing child %p", l->data);
225 e = l->data;
226 gst_bin_remove (bin, e);
227 gst_element_set_state (GST_ELEMENT (e), GST_STATE_NULL);
228 l = l->next;
229 };
230 g_list_free_full (childs, gst_object_unref);
231 }
232
233 static GstPadTemplate *
234 _get_compatible_sinkpad_template (GstTensorDecodeBin * self,
235 GstElementFactory * factory)
236 {
237 const GList *tpls;
238 GstPadTemplate *tpl;
239 const GList *fact_tpls;
240 GstPadTemplate *compa_sinkpad_tpl = NULL;
241 guint16 num_compa_sinkpad_template = 0, num_compa_srcpad_template = 0;
242
243 fact_tpls = gst_element_factory_get_static_pad_templates (factory);
244 for (tpls = fact_tpls; tpls; tpls = tpls->next) {
245 tpl = gst_static_pad_template_get (tpls->data);
246
247 /* FIXME: Add support for Request pads and Sometime pads */
248 if (tpl->presence != GST_PAD_ALWAYS) {
249 GST_WARNING_OBJECT (self, "Tensor decoder %s has %s pad which is "
250 "not currently supported by the tensordecodebin and is ignored.",
251 gst_element_factory_get_metadata (factory,
252 GST_ELEMENT_METADATA_LONGNAME),
253 tpl->presence == GST_PAD_REQUEST ? "request" : "sometimes");
254
255 /* Skip this template */
256 gst_clear_object (&tpl);
257 continue;
258 }
259
260 if (tpl->direction == GST_PAD_SINK) {
261 num_compa_sinkpad_template++;
262
263 if (num_compa_sinkpad_template == 1)
264 compa_sinkpad_tpl = gst_object_ref (tpl);
265
266 } else if (tpl->direction == GST_PAD_SRC) {
267 num_compa_srcpad_template++;
268 } else {
269 GST_WARNING_OBJECT (self,
270 "Tensor decoder %s has a pad template with UNKNOWN direction,"
271 " skipping this template.", gst_element_factory_get_metadata (factory,
272 GST_ELEMENT_METADATA_LONGNAME));
273
274 /* Skip this template */
275 gst_clear_object (&tpl);
276 continue;
277 }
278 gst_clear_object (&tpl);
279 }
280
281 /* FIXME: Add support for tensor decoder with multiple sinkpads and/or
282 * srcpads */
283 if (num_compa_sinkpad_template != 1 || num_compa_srcpad_template != 1) {
284 GST_WARNING_OBJECT (self,
285 "tensordecodebin only support tensor decoder with 1 always"
286 " sinkpad and 1 always srcpad, but %s has %u sinkpad and %u srcpad and will not be condiered",
287 gst_element_factory_get_metadata (factory,
288 GST_ELEMENT_METADATA_LONGNAME), num_compa_sinkpad_template,
289 num_compa_srcpad_template);
290 gst_clear_object (&compa_sinkpad_tpl);
291 }
292
293 return compa_sinkpad_tpl;
294 }
295
296 static gboolean
297 gst_tensordecodebin_sink_caps_event (GstTensorDecodeBin * self, GstCaps * ecaps)
298 {
299 gboolean ret = TRUE;
300 GstElement *e = NULL;
301 GstPad *sinkpad = NULL, *srcpad = NULL;
302 GstElementFactory *factory;
303 GList *factories = NULL;
304 GstCaps *tplcaps;
305
306 const GstStructure *s;
307 const GValue *v;
308
309 gst_ghost_pad_set_target (GST_GHOST_PAD (self->sinkpad), NULL);
310 gst_ghost_pad_set_target (GST_GHOST_PAD (self->srcpad), NULL);
311 _remove_all_elements (GST_BIN (self));
312 gst_caps_replace (&self->last_event_caps, ecaps);
313
314 /* We check all tensor group can be handled by a tensor decoder */
315 s = gst_caps_get_structure (ecaps, 0);
316 v = gst_structure_get_value (s, "tensors");
317
318 if (v == NULL) {
319 /* No tensor caps, we don't need any tensor decoder */
320 GST_INFO_OBJECT (self, "No tensor caps in, tensordecodebin will be "
321 "passthrough");
322 e = gst_element_factory_make ("identity", NULL);
323 if (!gst_bin_add (GST_BIN (self), e)) {
324 GST_ERROR_OBJECT (self, "Failed to add identity");
325 goto fail;
326 } else {
327 sinkpad = gst_element_get_static_pad (e, "sink");
328 if (!gst_ghost_pad_set_target (GST_GHOST_PAD (self->sinkpad), sinkpad)) {
329 GST_ERROR_OBJECT (self, "Failed to set sinkpad target to "
330 "identity.sinkpad");
331 goto fail;
332 }
333
334 gst_clear_object (&sinkpad);
335 srcpad = gst_element_get_static_pad (e, "src");
336 gst_element_sync_state_with_parent (e);
337
338 goto done;
339 }
340 }
341
342 /* NOTE: tensordecodebin assumes that tensordecoder does not modify the media
343 * or the capabilities. This is not a fundamental limitation of tensor
344 * capabilities but rather a limitation of the current tensordecodebin
345 * implementation. To implement support for tensordecoder-induced capability
346 * changes, we would need to maintain a full history of transformations. Currently,
347 * tensordecoder assumes the tensor was produced by inference on the attached
348 * media. However, this assumption will not hold if tensordecoder can modify
349 * media. Consequently, a tensordecoder following one that changes media would
350 * need to retrieve media details from the time the inference produced the
351 * tensor being decoded.
352 */
353 factories = gst_tensordecodebin_get_or_load_tensordec_factories (self);
354 for (GList * f = factories; f; f = g_list_next (f)) {
355 GstPadTemplate *compa_sinkpad_tpl = NULL;
356 factory = GST_ELEMENT_FACTORY (f->data);
357 compa_sinkpad_tpl = _get_compatible_sinkpad_template (self, factory);
358
359 tplcaps = gst_pad_template_get_caps (compa_sinkpad_tpl);
360
361 /* Check if sinkpad has at least a tensors field */
362 s = gst_caps_get_structure (tplcaps, 0);
363 if (!gst_structure_has_field (s, "tensors")) {
364 GST_WARNING_OBJECT (self,
365 "Element from %s factory have no tensors capabilities",
366 gst_element_factory_get_longname (factory));
367 gst_clear_caps (&tplcaps);
368 gst_clear_object (&compa_sinkpad_tpl);
369 continue;
370 }
371
372 if (gst_caps_is_subset (ecaps, tplcaps)) {
373 gst_clear_caps (&tplcaps);
374
375 e = gst_element_factory_create (factory, NULL);
376 sinkpad =
377 gst_element_get_static_pad (e, compa_sinkpad_tpl->name_template);
378 if (!sinkpad) {
379 GST_WARNING_OBJECT (self, "Element %p from %s factory has no sinkpad",
380 e, gst_element_factory_get_longname (factory));
381 gst_clear_object (&e);
382 gst_clear_object (&compa_sinkpad_tpl);
383
384 continue;
385 }
386 } else {
387 gst_clear_caps (&tplcaps);
388 gst_clear_object (&compa_sinkpad_tpl);
389 continue;
390 }
391
392 gst_clear_object (&compa_sinkpad_tpl);
393
394 if (gst_pad_query_accept_caps (sinkpad, ecaps)) {
395 gst_bin_add (GST_BIN (self), e);
396
397 GST_DEBUG_OBJECT (self, "selected tensor decoder: %" GST_PTR_FORMAT, e);
398
399 if (!gst_element_sync_state_with_parent (e)) {
400 GST_WARNING_OBJECT (self, "Element %" GST_PTR_FORMAT " failed to "
401 "synchronise its state with parent and will not be added to "
402 "this bin.", e);
403 gst_bin_remove (GST_BIN (self), e);
404 gst_clear_object (&e);
405 continue;
406 }
407
408 if (srcpad) {
409 if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) {
410 GST_ERROR_OBJECT (self,
411 "Could not link %" GST_PTR_FORMAT " and %" GST_PTR_FORMAT, srcpad,
412 sinkpad);
413 goto fail;
414 }
415
416 gst_clear_object (&srcpad);
417 } else {
418 if (!gst_ghost_pad_set_target (GST_GHOST_PAD (self->sinkpad), sinkpad)) {
419 GST_ERROR_OBJECT (self, "Failed to set sinkpad target");
420 goto fail;
421 }
422 }
423
424 gst_clear_object (&sinkpad);
425 srcpad = gst_element_get_static_pad (e, "src");
426
427 e = NULL;
428 continue;
429
430 } else {
431 GST_WARNING_OBJECT (self, "Factory (%p)'s sinkpad (%p) didn't accept "
432 "caps:%" GST_PTR_FORMAT, factory, sinkpad, ecaps);
433 gst_clear_object (&sinkpad);
434 gst_clear_object (&e);
435 }
436 }
437
438 g_list_free_full (g_steal_pointer (&factories), gst_object_unref);
439
440 if (srcpad == NULL) {
441 GST_WARNING_OBJECT (self, "Could not find tensor decoder for %"
442 GST_PTR_FORMAT, ecaps);
443 goto fail;
444 }
445
446 done:
447 if (!srcpad || !gst_ghost_pad_set_target (GST_GHOST_PAD (self->srcpad),
448 srcpad)) {
449 GST_ERROR_OBJECT (self, "Failed to set srcpad target");
450 goto fail;
451 }
452
453 gst_clear_object (&srcpad);
454 return ret;
455
456 fail:
457 g_list_free_full (factories, gst_object_unref);
458
459 _remove_all_elements (GST_BIN (self));
460
461 gst_clear_object (&srcpad);
462 gst_clear_object (&sinkpad);
463 gst_clear_object (&e);
464 return FALSE;
465 }
466
467 static gboolean
468 gst_tensordecodebin_sink_event (GstPad * pad, GstObject * parent, GstEvent *
469 event)
470 {
471 gboolean ret = TRUE;
472 GstTensorDecodeBin *self = GST_TENSORDECODEBIN (parent);
473 GstCaps *ecaps = NULL;
474
475 switch (GST_EVENT_TYPE (event)) {
476 case GST_EVENT_CAPS:
477 {
478 gst_event_parse_caps (event, &ecaps);
479
480 if (!gst_tensordecodebin_sink_caps_event (self, ecaps)) {
481 gst_caps_unref (ecaps);
482 goto done;
483 }
484
485 ret = gst_pad_event_default (pad, parent, event);
486 break;
487 }
488 default:
489 ret = gst_pad_event_default (pad, parent, event);
490 break;
491 }
492
493 done:
494 return ret;
495 }
496
497 static GstCaps *
498 _get_tensordecoders_caps (GstTensorDecodeBin * self)
499 {
500 GstElementFactory *factory;
501 GstCaps *tplcaps, *acc_caps = NULL;
502 GstPadTemplate *tpl;
503
504 GList *factories =
505 gst_tensordecodebin_get_or_load_tensordec_factories_unlocked (self);
506
507 if (self->aggregated_caps != NULL)
508 goto done;
509
510 acc_caps = gst_caps_new_empty ();
511
512 for (GList * f = factories; f; f = g_list_next (f)) {
513 factory = GST_ELEMENT_FACTORY (f->data);
514 tpl = _get_compatible_sinkpad_template (self, factory);
515
516 if (!tpl) {
517 GST_WARNING_OBJECT (self,
518 "No compatible sinkpad template found %s factory",
519 gst_element_factory_get_metadata (factory,
520 GST_ELEMENT_METADATA_LONGNAME));
521 continue;
522 }
523
524 tplcaps = gst_pad_template_get_caps (tpl);
525 gst_clear_object (&tpl);
526 acc_caps = gst_caps_merge (acc_caps, tplcaps);
527 }
528
529 self->aggregated_caps = acc_caps;
530
531 done:
532 g_list_free_full (factories, gst_object_unref);
533
534 return gst_caps_ref (self->aggregated_caps);
535 }
536
537 static gboolean
538 gst_tensordecodebin_sink_query (GstPad * pad, GstObject * parent, GstQuery *
539 query)
540 {
541 gboolean ret;
542 GstTensorDecodeBin *self = GST_TENSORDECODEBIN (parent);
543
544 switch (query->type) {
545 case GST_QUERY_CAPS:
546 {
547 GstCaps *acc_caps, *filter_caps = NULL, *intersection;
548 GstQuery *dn_query;
549 gst_query_parse_caps (query, &filter_caps);
550
551 GST_OBJECT_LOCK (self);
552 acc_caps = _get_tensordecoders_caps (self);
553 GST_OBJECT_UNLOCK (self);
554
555 if (filter_caps) {
556 intersection = gst_caps_intersect (acc_caps, filter_caps);
557 gst_caps_replace (&acc_caps, intersection);
558 }
559
560 dn_query = gst_query_new_caps (acc_caps);
561 if ((ret = gst_pad_peer_query (self->srcpad, dn_query))) {
562 gst_query_parse_caps (dn_query, &filter_caps);
563 if (filter_caps) {
564 intersection = gst_caps_intersect (acc_caps, filter_caps);
565 gst_caps_replace (&acc_caps, intersection);
566 }
567 }
568
569 gst_clear_query (&dn_query);
570 gst_query_set_caps_result (query, acc_caps);
571 gst_caps_unref (acc_caps);
572 break;
573 }
574 case GST_QUERY_ACCEPT_CAPS:
575 {
576 GstCaps *caps, *acc_caps = NULL;
577
578 GST_OBJECT_LOCK (self);
579 acc_caps = _get_tensordecoders_caps (self);
580 gst_query_parse_accept_caps (query, &caps);
581 gst_query_set_accept_caps_result (query, gst_caps_can_intersect (acc_caps,
582 caps));
583 GST_OBJECT_UNLOCK (self);
584
585 gst_caps_unref (acc_caps);
586 ret = TRUE;
587 break;
588 }
589 default:
590 ret = gst_pad_query_default (pad, parent, query);
591 break;
592 }
593
594 return ret;
595 }