Debian Codegraph @

gst/tensordecoders/gstclassifiertensordecoder.c

raw
1 /*
2 * GStreamer gstreamer-classifiertensordecoder
3 * Copyright (C) 2025 Collabora Ltd.
4 * @author: Daniel Morin <[email protected]>
5 *
6 * gstclassifiertensordecoder.c
7 *
8 * This library is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Library General Public
10 * License as published by the Free Software Foundation; either
11 * version 2 of the License, or (at your option) any later version.
12 *
13 * This library is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Library General Public License for more details.
17 *
18 * You should have received a copy of the GNU Library General Public
19 * License along with this library; if not, write to the
20 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
21 * Boston, MA 02110-1301, USA.
22 */
23
24 /**
25 * SECTION:element-classifiertensordecoder.c
26 * @short_description: Decode tensors from classification model using a common
27 * tensor output format.
28 *
29 *
30 * This element can parse per-buffer inference tensor meta data generated by
31 * an upstream inference element.
32 *
33 * Tensor format must be:
34 * Dims: [batch-size, class_count]
35 * Datatype: float32
36 *
37 * Tensor [M,N]
38 * Batch 0 | Class 0 confidence level | ... | Class N confidence level |
39 * ...
40 * Batch M | Class 0 confidence level | ... | Class N confidence level |
41 *
42 * In-memory tensor format:
43 *
44 * |Batch 0, Class 0 confidence level |
45 * |Batch 0, ... |
46 * |Batch 0, Class N confidence level |
47 * | ... |
48 * |Batch M, Class 0 confidence level |
49 * |Batch M, ... |
50 * |Batch M, Class N confidence level |
51 *
52 *
53 * ## Example launch command:
54 * |[
55 * gst-launch-1.0 filesrc location=/onnx-models/images/bus.jpg \
56 * ! jpegdec \
57 * ! videoconvertscale add-borders=1 \
58 * ! onnxinference execution-provider=cpu \
59 * model-file=/onnx-models/models/mobilenet_v1.onnx \
60 * ! classifiertensordecoder labels-file=labels.txt ! fakesink \
61 * ]| This pipeline create an tensor-decoder for classification model
62 *
63 */
64
65 #ifdef HAVE_CONFIG_H
66 #include "config.h"
67 #endif
68
69 #include "gstclassifiertensordecoder.h"
70 #include <gst/gst.h>
71 #include <math.h>
72 #include <gst/analytics/analytics.h>
73
74 #define GROUP_ID_CLASSIFICATION "classification-generic-out"
75 #define GROUP_ID_CLASSIFICATION_SOFTMAXED "classification-generic-softmaxed-out"
76 #define GST_MODEL_STD_IMAGE_CLASSIFICATION "classification-generic-out"
77 #define GST_MODEL_STD_IMAGE_CLASSIFICATION_SOFTMAXED "classification-generic-softmaxed-out"
78
79 GST_DEBUG_CATEGORY_STATIC (classifier_tensor_decoder_debug);
80 #define GST_CAT_DEFAULT classifier_tensor_decoder_debug
81 #define gst_classifier_tensor_decoder_parent_class parent_class
82
83 GST_ELEMENT_REGISTER_DEFINE (classifier_tensor_decoder,
84 "classifiertensordecoder", GST_RANK_SECONDARY,
85 GST_TYPE_CLASSIFIER_TENSOR_DECODER);
86
87
88 /* GstClassifierTensorDecoder properties */
89 enum
90 {
91 PROP_0,
92 PROP_THRESHOLD,
93 PROP_LABEL_FILE
94 };
95
96 static const float DEFAULT_THRESHOLD = 0.7f;
97
98 static GstStaticPadTemplate gst_classifier_tensor_decoder_src_template =
99 GST_STATIC_PAD_TEMPLATE ("src",
100 GST_PAD_SRC,
101 GST_PAD_ALWAYS,
102 GST_STATIC_CAPS_ANY);
103
104 /* *INDENT-OFF* */
105
106 static GstStaticPadTemplate gst_classifier_tensor_decoder_sink_template =
107 GST_STATIC_PAD_TEMPLATE ("sink",
108 GST_PAD_SINK,
109 GST_PAD_ALWAYS,
110 GST_STATIC_CAPS (
111 "video/x-raw,"
112 "tensors=(structure)["
113 "tensorgroups,"
114 GROUP_ID_CLASSIFICATION"=(/uniquelist){"
115 "(GstCaps)["
116 "tensor/strided,"
117 "tensor-id="GST_MODEL_STD_IMAGE_CLASSIFICATION","
118 "dims=<(int)[0,1], (int)[1,max]>,"
119 "dims-order=(string)row-major,"
120 "type={float32, uint8};"
121 "tensor/strided,"
122 "tensor-id="GST_MODEL_STD_IMAGE_CLASSIFICATION","
123 "dims=<(int)[1,max]>,"
124 "dims-order=(string)row-major,"
125 "type={float32, uint8};]"
126 "}"
127 "];"
128 "video/x-raw,"
129 "tensors=(structure)["
130 "tensorgroups,"
131 GROUP_ID_CLASSIFICATION_SOFTMAXED"=(/uniquelist){"
132 "(GstCaps)["
133 "tensor/strided,"
134 "tensor-id="GST_MODEL_STD_IMAGE_CLASSIFICATION_SOFTMAXED","
135 "dims=<(int)[0,1], (int)[1,max]>,"
136 "dims-order=(string)row-major,"
137 "type={float32, uint8};"
138 "tensor/strided,"
139 "tensor-id="GST_MODEL_STD_IMAGE_CLASSIFICATION_SOFTMAXED","
140 "dims=<(int)[1,max]>,"
141 "dims-order=(string)row-major,"
142 "type={float32, uint8};]"
143 "}"
144 "]"
145 ));
146 /* *INDENT-ON* */
147
148 static void gst_classifier_tensor_decoder_set_property (GObject * object,
149 guint prop_id, const GValue * value, GParamSpec * pspec);
150 static void gst_classifier_tensor_decoder_get_property (GObject * object,
151 guint prop_id, GValue * value, GParamSpec * pspec);
152
153 static void gst_classifier_tensor_decoder_finalize (GObject * object);
154
155 static GstFlowReturn
156 gst_classifier_tensor_decoder_transform_ip (GstBaseTransform * trans,
157 GstBuffer * buf);
158
159 static GstStateChangeReturn
160 gst_classifier_tensor_decoder_change_state (GstElement * element,
161 GstStateChange transition);
162
163 static gboolean
164 gst_classifier_tensor_decoder_set_caps (GstBaseTransform * trans,
165 GstCaps * incaps, GstCaps * outcaps);
166
167
168 #define softmax(len, values, results, max_val) \
169 gsize i; \
170 gfloat sum = 0.0; \
171 gfloat value; \
172 g_return_if_fail (values != NULL); \
173 g_return_if_fail (results != NULL); \
174 \
175 /* Calculate exponential of every value */ \
176 for (i = 0; i < len; i++) { \
177 value = values[i] / max_val; \
178 results[i] = exp (value); \
179 sum += results[i]; \
180 } \
181 \
182 /* Complete softmax */ \
183 for (i = 0; i < len; i++) { \
184 result[i] = results[i] / sum; \
185 }
186
187 static void
188 softmax_u8 (gsize len, const guint8 * values, gfloat * result)
189 {
190 softmax (len, values, result, 255.0);
191 }
192
193 static void
194 softmax_f32 (gsize len, const gfloat * values, gfloat * result)
195 {
196 softmax (len, values, result, 1.0);
197 }
198
199 G_DEFINE_TYPE (GstClassifierTensorDecoder, gst_classifier_tensor_decoder,
200 GST_TYPE_BASE_TRANSFORM);
201
202 static void
203 gst_classifier_tensor_decoder_class_init (GstClassifierTensorDecoderClass *
204 klass)
205 {
206 GObjectClass *gobject_class = (GObjectClass *) klass;
207 GstElementClass *element_class = (GstElementClass *) klass;
208 GstBaseTransformClass *basetransform_class = (GstBaseTransformClass *) klass;
209
210 GST_DEBUG_CATEGORY_INIT (classifier_tensor_decoder_debug,
211 "classifiertensordecoder", 0,
212 "Tensor decoder for classification model with common output format");
213
214 gobject_class->set_property = gst_classifier_tensor_decoder_set_property;
215 gobject_class->get_property = gst_classifier_tensor_decoder_get_property;
216 gobject_class->finalize = gst_classifier_tensor_decoder_finalize;
217
218 g_object_class_install_property (G_OBJECT_CLASS (klass),
219 PROP_THRESHOLD,
220 g_param_spec_float ("class-confidence-threshold",
221 "Class confidence threshold",
222 "Classes with a confidence level inferior to this threshold "
223 "will be excluded",
224 0.0, 1.0, DEFAULT_THRESHOLD,
225 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
226
227 g_object_class_install_property (G_OBJECT_CLASS (klass),
228 PROP_LABEL_FILE,
229 g_param_spec_string ("labels-file",
230 "Class labels file",
231 "Path to a file containing class label. COCO format",
232 NULL, (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
233
234 element_class->change_state = gst_classifier_tensor_decoder_change_state;
235
236 gst_element_class_set_static_metadata (element_class,
237 "Classification tensor decoder", "Tensordecoder",
238 "Decode tensors output from classification model using common format.\n"
239 "\tTensor format must be: \n" "\t\tDims: [batch-size, class_count]\n"
240 "\t\tDatatype: float32 \n" "\n" "\t\tTensor [M,N]\n"
241 "\t\t\tBatch 0 | Class 0 confidence level | ... | Class N-1 confidence level |\n"
242 "\t\t\t...\n"
243 "\t\t\tBatch M-1 | Class 0 confidence level | ... | Class N-1 confidence level |\n"
244 "\t\t\n" "\tIn-memory tensor format:\n" "\n"
245 "\t\t|Batch 0, Class 0 confidence level |\n"
246 "\t\t|Batch 0, ... |\n"
247 "\t\t|Batch 0, Class N-1 confidence level |\n"
248 "\t\t| ... |\n"
249 "\t\t|Batch M-1, Class 0 confidence level |\n"
250 "\t\t|Batch M-1, ... |\n"
251 "\t\t|Batch M-1, Class N-1 confidence level |\n" "\n" " model",
252 "Daniel Morin <[email protected]>");
253
254 gst_element_class_add_pad_template (element_class,
255 gst_static_pad_template_get
256 (&gst_classifier_tensor_decoder_sink_template));
257
258 gst_element_class_add_pad_template (element_class,
259 gst_static_pad_template_get
260 (&gst_classifier_tensor_decoder_src_template));
261
262 basetransform_class->transform_ip =
263 GST_DEBUG_FUNCPTR (gst_classifier_tensor_decoder_transform_ip);
264
265 basetransform_class->set_caps =
266 GST_DEBUG_FUNCPTR (gst_classifier_tensor_decoder_set_caps);
267 }
268
269 static void
270 gst_classifier_tensor_decoder_init (GstClassifierTensorDecoder * self)
271 {
272 self->threshold = DEFAULT_THRESHOLD;
273 self->labels_file = NULL;
274 self->postproc_result = NULL;
275 self->class_count = 0;
276 self->do_softmax = TRUE;
277
278 gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (self), FALSE);
279 GST_PAD_UNSET_ACCEPT_INTERSECT (self->basetransform.sinkpad);
280 }
281
282 static void
283 gst_classifier_tensor_decoder_finalize (GObject * object)
284 {
285 GstClassifierTensorDecoder *self = GST_CLASSIFIER_TENSOR_DECODER (object);
286
287 g_free (self->labels_file);
288 G_OBJECT_CLASS (gst_classifier_tensor_decoder_parent_class)->finalize
289 (object);
290 }
291
292 static void
293 gst_classifier_tensor_decoder_set_property (GObject * object, guint prop_id,
294 const GValue * value, GParamSpec * pspec)
295 {
296 GstClassifierTensorDecoder *self = GST_CLASSIFIER_TENSOR_DECODER (object);
297 static GFileTest filetest = (G_FILE_TEST_EXISTS | G_FILE_TEST_IS_REGULAR);
298
299 switch (prop_id) {
300 case PROP_THRESHOLD:
301 self->threshold = g_value_get_float (value);
302 break;
303 case PROP_LABEL_FILE:
304 self->labels_file = g_strdup (g_value_get_string (value));
305
306 if (self->labels_file) {
307 if (!g_file_test (self->labels_file, filetest)) {
308 GST_ERROR_OBJECT (self, "Unable to load %s", self->labels_file);
309 g_free (g_steal_pointer (&self->labels_file));
310 }
311 } else {
312 GST_ERROR_OBJECT (self, "Invalid file");
313 }
314 break;
315 default:
316 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
317 break;
318 }
319 }
320
321 static void
322 gst_classifier_tensor_decoder_get_property (GObject * object, guint prop_id,
323 GValue * value, GParamSpec * pspec)
324 {
325 GstClassifierTensorDecoder *self = GST_CLASSIFIER_TENSOR_DECODER (object);
326
327 switch (prop_id) {
328 case PROP_THRESHOLD:
329 g_value_set_float (value, self->threshold);
330 break;
331 case PROP_LABEL_FILE:
332 g_value_set_string (value, self->labels_file);
333 break;
334 default:
335 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
336 break;
337 }
338 }
339
340 static guint
341 gst_classifier_tensor_decoder_load_labels (GstClassifierTensorDecoder * self)
342 {
343 gchar *content = NULL;
344 gchar **tokens = NULL;
345 gsize len;
346 GError *err = NULL;
347 GQuark val;
348 GArray *class_quark = NULL;
349
350 if (self->labels_file == NULL) {
351 GST_ERROR_OBJECT (self, "Missing label file");
352 return 0;
353 }
354 if (!g_file_get_contents (self->labels_file, &content, &len, &err)) {
355 GST_ERROR_OBJECT (self, "Could not load labels file %s: %s",
356 self->labels_file, err->message);
357 g_error_free (err);
358 return 0;
359 }
360
361 if (len == 0) {
362 GST_ERROR_OBJECT (self, "Labels file %s is empty", self->labels_file);
363 g_free (content);
364 return 0;
365 }
366
367 tokens = g_strsplit (content, "\n", 0);
368 g_free (content);
369
370 if (tokens[0] != NULL) {
371 class_quark =
372 g_array_sized_new (FALSE, FALSE, sizeof (GQuark), self->class_count);
373 }
374
375 self->class_quark = g_array_new (FALSE, FALSE, sizeof (GQuark));
376
377 for (int i = 0; tokens[i] != NULL && tokens[i][0] != '\0'; i++) {
378 val = g_quark_from_string (tokens[i]);
379 g_array_append_val (class_quark, val);
380 }
381
382 if (class_quark == NULL)
383 GST_WARNING_OBJECT (self, "Label %s file does not contain any labels",
384 self->labels_file);
385
386 self->class_quark = class_quark;
387
388 g_strfreev (tokens);
389 return self->class_quark->len;
390 }
391
392 static GstStateChangeReturn
393 gst_classifier_tensor_decoder_change_state (GstElement * element,
394 GstStateChange transition)
395 {
396 GstClassifierTensorDecoder *self = GST_CLASSIFIER_TENSOR_DECODER (element);
397 GstStateChangeReturn ret;
398
399 switch (transition) {
400 case GST_STATE_CHANGE_NULL_TO_READY:
401 if (self->labels_file != NULL &&
402 !gst_classifier_tensor_decoder_load_labels (self)) {
403 return GST_STATE_CHANGE_FAILURE;
404 }
405 break;
406 default:
407 break;
408 }
409
410 ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
411
412 switch (transition) {
413 case GST_STATE_CHANGE_READY_TO_NULL:
414 if (self->class_quark)
415 g_array_free (self->class_quark, FALSE);
416 if (self->postproc_result)
417 g_array_free (self->postproc_result, TRUE);
418 break;
419 default:
420 break;
421 }
422
423 return ret;
424 }
425
426 static const GstTensor *
427 get_tensor (GstTensorMeta * tmeta, GQuark tensor_id)
428 {
429 const GstTensor *tensor;
430 const gsize DIMS[] = { 1, G_MAXSIZE };
431
432 tensor = gst_tensor_meta_get_typed_tensor (tmeta, tensor_id,
433 GST_TENSOR_DATA_TYPE_FLOAT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 1, NULL);
434 if (tensor == NULL)
435 tensor = gst_tensor_meta_get_typed_tensor (tmeta, tensor_id,
436 GST_TENSOR_DATA_TYPE_FLOAT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 2, DIMS);
437 if (tensor == NULL)
438 tensor = gst_tensor_meta_get_typed_tensor (tmeta, tensor_id,
439 GST_TENSOR_DATA_TYPE_UINT8, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 1, NULL);
440 if (tensor == NULL)
441 tensor = gst_tensor_meta_get_typed_tensor (tmeta, tensor_id,
442 GST_TENSOR_DATA_TYPE_UINT8, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 2, DIMS);
443
444 return tensor;
445 }
446
447 static const GstTensor *
448 gst_classifier_tensor_decoder_get_tensor (GstClassifierTensorDecoder *
449 self, GstBuffer * buf)
450 {
451 GstMeta *meta = NULL;
452 gpointer iter_state = NULL;
453 const gchar *expected_tensor_id;
454
455 if (!gst_buffer_get_meta (buf, GST_TENSOR_META_API_TYPE)) {
456 GST_DEBUG_OBJECT (self,
457 "missing tensor meta from buffer %" GST_PTR_FORMAT, buf);
458 return NULL;
459 }
460
461 /* Use the tensor-id that matches what was negotiated */
462 expected_tensor_id = self->do_softmax ?
463 GST_MODEL_STD_IMAGE_CLASSIFICATION :
464 GST_MODEL_STD_IMAGE_CLASSIFICATION_SOFTMAXED;
465
466 while ((meta = gst_buffer_iterate_meta_filtered (buf, &iter_state,
467 GST_TENSOR_META_API_TYPE))) {
468 GstTensorMeta *tensor_meta = (GstTensorMeta *) meta;
469 const GstTensor *tensor;
470
471 tensor = get_tensor (tensor_meta,
472 g_quark_from_static_string (expected_tensor_id));
473
474 if (tensor)
475 return tensor;
476 }
477
478 return NULL;
479 }
480
481 static GstFlowReturn
482 gst_classifier_tensor_decoder_decode (GstClassifierTensorDecoder * self,
483 const GstTensor * tensor, GstAnalyticsRelationMeta * rmeta)
484 {
485 GstMapInfo map_info = GST_MAP_INFO_INIT;
486 gfloat max = 0.0;
487 gfloat *result_data = NULL;
488 gsize len;
489 GQuark q, qmax = 0;
490 gint max_idx = -1;
491 GstAnalyticsClsMtd cls_mtd;
492
493 len = tensor->dims[tensor->num_dims - 1];
494
495 if (len != self->class_quark->len) {
496 GST_WARNING_OBJECT (self, "Labels file has size %zu, but the tensor has"
497 " %u entries, it is probably not the right labels file",
498 len, self->class_quark->len);
499 len = MIN (len, self->class_quark->len);
500 }
501
502 if (!gst_buffer_map (tensor->data, &map_info, GST_MAP_READ)) {
503 GST_ELEMENT_ERROR (self, STREAM, FAILED, (NULL),
504 ("Failed to map tensor data"));
505 return GST_FLOW_ERROR;
506 }
507
508 GST_TRACE_OBJECT (self, "Tensor shape dims %zu", tensor->num_dims);
509
510 if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_TRACE) {
511 for (gint i = 0; i < tensor->num_dims; i++) {
512 GST_TRACE_OBJECT (self, "Tensor dim %d: %zu", i, tensor->dims[i]);
513 }
514 }
515
516 switch (tensor->data_type) {
517 case GST_TENSOR_DATA_TYPE_FLOAT32:
518 if (map_info.size != len * sizeof (gfloat)) {
519 GST_ELEMENT_ERROR (self, STREAM, FAILED, (NULL),
520 ("Tensor size is not as expected for float: map.size(%zu) !="
521 " label-file-length(%zu) * sizeof(float)(%zu)", map_info.size,
522 len, sizeof (float)));
523 goto error_mapped;
524 }
525
526 if (self->do_softmax) {
527 result_data = (gfloat *) self->postproc_result->data;
528 softmax_f32 (len, (gfloat *) map_info.data, result_data);
529 } else {
530 /* Already softmaxed, use data directly */
531 result_data = (gfloat *) map_info.data;
532 }
533 break;
534 case GST_TENSOR_DATA_TYPE_UINT8:
535 if (map_info.size != len) {
536 GST_ELEMENT_ERROR (self, STREAM, FAILED, (NULL),
537 ("Tensor size is not as expected for uint8: map.size(%zu) !="
538 " label-file-length(%zu))", map_info.size, len));
539 goto error_mapped;
540 }
541
542 /* Always need conversion buffer for uint8 -> float */
543 result_data = (gfloat *) self->postproc_result->data;
544 if (self->do_softmax) {
545 softmax_u8 (len, (guint8 *) map_info.data, result_data);
546 } else {
547 const guint8 *uint8_data = map_info.data;
548 for (gint i = 0; i < len; i++) {
549 result_data[i] = uint8_data[i] / 255.0;
550 }
551 }
552 break;
553 default:
554 GST_ELEMENT_ERROR (self, STREAM, FAILED, (NULL),
555 ("Can't handle data type %d", tensor->data_type));
556 goto error_mapped;
557 }
558
559 for (gint j = 0; j < len; j++) {
560 q = g_array_index (self->class_quark, GQuark, j);
561
562 if (result_data[j] > max) {
563 max = result_data[j];
564 max_idx = j;
565 qmax = q;
566 }
567 }
568
569 gst_buffer_unmap (tensor->data, &map_info);
570
571 if (max_idx != -1) {
572 gst_analytics_relation_meta_add_one_cls_mtd (rmeta, max, qmax, &cls_mtd);
573 GST_LOG_OBJECT (self, "Max class is %d:%s with %f", max_idx,
574 g_quark_to_string (qmax), max);
575 }
576
577 return GST_FLOW_OK;
578
579 error_mapped:
580 gst_buffer_unmap (tensor->data, &map_info);
581 return GST_FLOW_ERROR;
582 }
583
584 static GstFlowReturn
585 gst_classifier_tensor_decoder_transform_ip (GstBaseTransform * trans,
586 GstBuffer * buf)
587 {
588 GstClassifierTensorDecoder *self = GST_CLASSIFIER_TENSOR_DECODER (trans);
589 const GstTensor *tensor;
590 GstAnalyticsRelationMeta *rmeta;
591
592 tensor = gst_classifier_tensor_decoder_get_tensor (self, buf);
593 if (tensor == NULL) {
594 GST_WARNING_OBJECT (trans, "missing tensor meta");
595 return GST_FLOW_OK;
596 }
597
598 rmeta = gst_buffer_add_analytics_relation_meta (buf);
599
600 return gst_classifier_tensor_decoder_decode (self, tensor, rmeta);
601 }
602
603 static gboolean
604 gst_classifier_tensor_decoder_set_caps (GstBaseTransform * trans,
605 GstCaps * incaps, GstCaps * outcaps)
606 {
607 GstClassifierTensorDecoder *self = GST_CLASSIFIER_TENSOR_DECODER (trans);
608 const GstCaps *tcaps;
609 const GstStructure *s, *ts, *dims_s;
610 const GValue *dims_v, *dim_v, *tensors_v, *tensors_gv, *tensor_caps_v;
611 gsize dims_size, batchsize = 1;
612 gchar buffer[32];
613 GQuark val;
614
615 /* Get the classification tensor */
616 s = gst_caps_get_structure (incaps, 0);
617 g_return_val_if_fail (s != NULL, FALSE);
618
619 tensors_v = gst_structure_get_value (s, "tensors");
620 g_return_val_if_fail (tensors_v != NULL, FALSE);
621
622 ts = gst_value_get_structure (tensors_v);
623 g_return_val_if_fail (ts != NULL, FALSE);
624
625 /* Try to get classification group (non-softmaxed) first */
626 tensors_gv = gst_structure_get_value (ts, GROUP_ID_CLASSIFICATION);
627 /* If not found, try softmaxed group */
628 if (tensors_gv == NULL)
629 tensors_gv =
630 gst_structure_get_value (ts, GROUP_ID_CLASSIFICATION_SOFTMAXED);
631 g_return_val_if_fail (tensors_gv != NULL, FALSE);
632
633 tensor_caps_v = gst_value_unique_list_get_value (tensors_gv, 0);
634 g_return_val_if_fail (tensor_caps_v != NULL, FALSE);
635
636 tcaps = gst_value_get_caps (tensor_caps_v);
637 s = gst_caps_get_structure (tcaps, 0);
638 g_return_val_if_fail (tcaps != NULL, FALSE);
639
640 if (gst_structure_has_field (s, "tensor-id")) {
641 const gchar *tensor_id = gst_structure_get_string (s, "tensor-id");
642
643 /* Determine if we need to apply softmax based on negotiated tensor-id */
644 if (g_strcmp0 (tensor_id, GST_MODEL_STD_IMAGE_CLASSIFICATION) == 0) {
645 self->do_softmax = TRUE;
646 } else if (g_strcmp0 (tensor_id,
647 GST_MODEL_STD_IMAGE_CLASSIFICATION_SOFTMAXED) == 0) {
648 self->do_softmax = FALSE;
649 } else {
650 /* Unknown tensor-id, skip */
651 return TRUE;
652 }
653
654 dims_s = gst_caps_get_structure (tcaps, 0);
655 dims_v = gst_structure_get_value (dims_s, "dims");
656 dims_size = gst_value_array_get_size (dims_v);
657
658 if (dims_size == 2) {
659 /* Explicit batch-size */
660 dim_v = gst_value_array_get_value (dims_v, 0);
661 batchsize = g_value_get_int (dim_v);
662
663 if (batchsize == 0)
664 batchsize = 1;
665
666 dim_v = gst_value_array_get_value (dims_v, 1);
667 } else {
668 dim_v = gst_value_array_get_value (dims_v, 0);
669 }
670
671 /* Get classes count */
672 self->class_count = g_value_get_int (dim_v);
673
674 /* Allocate postproc_result buffer for softmax or uint8->float conversion */
675 self->postproc_result =
676 g_array_sized_new (FALSE, TRUE, sizeof (gfloat), self->class_count);
677
678 if (self->class_quark != NULL &&
679 self->class_count != self->class_quark->len) {
680 GST_ELEMENT_ERROR (GST_BASE_TRANSFORM (self), STREAM, FAILED,
681 ("Label-file/Tensor mismatch"),
682 ("Class count from tensor mismatch class count from label file"));
683 return FALSE;
684 }
685
686 /* Generate labels if no label file was specified. */
687 if (self->class_quark == NULL) {
688 self->class_quark = g_array_sized_new (FALSE, FALSE, sizeof (GQuark),
689 self->class_count);
690 for (gsize i = 0; i < self->class_count; i++) {
691 if (g_snprintf (buffer, sizeof (buffer), "%zu", i) >= sizeof (buffer)) {
692 g_array_free (self->postproc_result, FALSE);
693 self->postproc_result = NULL;
694 return FALSE;
695 }
696 val = g_quark_from_string (buffer);
697 g_array_append_val (self->class_quark, val);
698 }
699 }
700 }
701
702 return TRUE;
703 }