Debian Codegraph @

gst/tensordecoders/gstyolosegtensordecoder.c

raw
1 /*
2 * GStreamer
3 * Copyright (C) 2024 Collabora Ltd.
4 * Authors: Daniel Morin <[email protected]>
5 * Vineet Suryan <[email protected]>
6 * Santosh Mahto <[email protected]>
7 *
8 * This library is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Library General Public
10 * License as published by the Free Software Foundation; either
11 * version 2 of the License, or (at your option) any later version.
12 *
13 * This library is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Library General Public License for more details.
17 *
18 * You should have received a copy of the GNU Library General Public
19 * License along with this library; if not, write to the
20 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
21 * Boston, MA 02110-1301, USA.
22 */
23
24 /**
25 * SECTION:element-yolosegv8tensordec
26 * @short_description: Decode tensors from a FastSAM/YOLOv8 segmentation
27 * models
28 *
29 * This element can parse per-buffer inference tensors meta data generated by an upstream
30 * inference element
31 *
32 * ## Example launch command:
33 *
34 * Test image file, model file and labels file can be found here :
35 * https://gitlab.collabora.com/gstreamer/onnx-models
36 *
37 * gst-launch-1.0 v4l2src device=/dev/video4 ! videorate max-rate=3 \
38 * ! videoconvertscale ! video/x-raw, pixel-aspect-ratio=1/1 \
39 * ! onnxinference \
40 * model-file=/home/dmorin/repos/onnx-models/models/yolov8s-seg.onnx \
41 * ! yolosegv8tensordec class-confidence-threshold=0.8 iou-threshold=0.3 \
42 * max-detections=100 \
43 * label-file=/home/dmorin/repos/onnx-models/labels/COCO_classes.txt \
44 * ! segmentationoverlay \
45 * ! glimagesink sink="gtkglsink processing-deadline=300000000
46 *
47 * The original repository of the Yolo is located at
48 * https://github.com/ultralytics/ultralytics.
49 * For easy experimentation, a object segmentation model based on Yolo
50 * architecture in Onnx format can be found at https://col.la/gstonnxmodelseg.
51 * This model already has the required tensor-ids embedded in the model
52 * It's also possible to embed tensor-ids into any model based on Yolo
53 * architecture to allow this tensor-decoder to decode tensors. This process
54 * is described in the Readme of this repository: https://col.la/gstonnxmodels"
55 *
56 * Since: 1.28
57 */
58
59 #ifdef HAVE_CONFI_H
60 #include "config.h"
61 #endif
62
63 #include "gstyolosegtensordecoder.h"
64
65 #include <gst/analytics/analytics.h>
66 #include <gio/gio.h>
67
68 #include <math.h>
69
70 #define YOLO_SEGMENTATION_LOGITS "yolo-v8-segmentation-out-protos"
71 GQuark YOLO_SEGMENTATION_LOGITS_TENSOR_ID;
72
73 #define YOLO_SEGMENTATION_DETECTION_MASK "yolo-v8-segmentation-out-detections"
74 GQuark YOLO_SEGMENTATION_DETECTION_MASK_ID;
75
76 /* *INDENT-OFF* */
77 static GstStaticPadTemplate gst_yolo_seg_tensor_decoder_sink_template =
78 GST_STATIC_PAD_TEMPLATE ("sink",
79 GST_PAD_SINK,
80 GST_PAD_ALWAYS,
81 GST_STATIC_CAPS ("video/x-raw,"
82 "tensors=(structure)["
83 "tensorgroups,"
84 "yolo-v8-segmentation-out=(/uniquelist){"
85 "(GstCaps)["
86 "tensor/strided,"
87 "tensor-id=yolo-v8-segmentation-out-detections,"
88 "dims=(int)<1, [1,max], [1,max]>,"
89 "dims-order=(string)col-major,"
90 "type=(string)float32"
91 "],"
92 "(GstCaps)["
93 "tensor/strided,"
94 "tensor-id=yolo-v8-segmentation-out-protos,"
95 "dims=(int)<1, [1,max], [1,max], [1,max]>,"
96 "dims-order=(string)col-major,"
97 "type=(string)float32"
98 "]"
99 "}"
100 "]"
101 ));
102 /* *INDENT-ON* */
103
104
105 GST_DEBUG_CATEGORY_STATIC (yolo_seg_tensor_decoder_debug);
106 #define GST_CAT_DEFAULT yolo_seg_tensor_decoder_debug
107
108 GST_ELEMENT_REGISTER_DEFINE (yolo_seg_tensor_decoder, "yolosegv8tensordec",
109 GST_RANK_SECONDARY, GST_TYPE_YOLO_SEG_TENSOR_DECODER);
110
111 /* For debug purpose */
112 typedef struct _DebugCandidates
113 {
114 gpointer self;
115 gsize fields; /* Fields count do debug */
116 gsize offset; /* Fields offset */
117 gsize start; /* First field index to debug */
118 } DebugCandidates;
119
120 /* GstYoloSegTensorDecoder Prototypes */
121 static gboolean gst_yolo_seg_tensor_decoder_stop (GstBaseTransform * trans);
122 static GstFlowReturn gst_yolo_seg_tensor_decoder_transform_ip (GstBaseTransform
123 * trans, GstBuffer * buf);
124
125 static void gst_yolo_seg_tensor_decoder_object_found (GstYoloTensorDecoder * od,
126 GstAnalyticsRelationMeta * rmeta, BBox * bb, gfloat confidence,
127 GQuark class_quark, const gfloat * candidate_masks, gsize offset,
128 guint count);
129
130 G_DEFINE_TYPE (GstYoloSegTensorDecoder, gst_yolo_seg_tensor_decoder,
131 GST_TYPE_YOLO_TENSOR_DECODER);
132
133 static gboolean
134 gst_yolo_seg_tensor_decoder_stop (GstBaseTransform * trans)
135 {
136 GstYoloSegTensorDecoder *self = GST_YOLO_SEG_TENSOR_DECODER (trans);
137
138 self->mask_w = 0;
139 self->mask_h = 0;
140 self->mask_length = 0;
141 if (self->mask_pool)
142 gst_buffer_pool_set_active (self->mask_pool, FALSE);
143 g_clear_object (&self->mask_pool);
144
145 return TRUE;
146 }
147
148 static void
149 gst_yolo_seg_tensor_decoder_class_init (GstYoloSegTensorDecoderClass * klass)
150 {
151 GstElementClass *element_class = (GstElementClass *) klass;
152 GstBaseTransformClass *basetransform_class = (GstBaseTransformClass *) klass;
153 GstYoloTensorDecoderClass *od_class = (GstYoloTensorDecoderClass *) klass;
154
155 /* Define GstYoloSegTensorDecoder debug category. */
156 GST_DEBUG_CATEGORY_INIT (yolo_seg_tensor_decoder_debug,
157 "yolosegv8tensordec", 0, "Tensor decoder for Yolo segmentation models");
158
159 YOLO_SEGMENTATION_DETECTION_MASK_ID =
160 g_quark_from_static_string (YOLO_SEGMENTATION_DETECTION_MASK);
161 YOLO_SEGMENTATION_LOGITS_TENSOR_ID =
162 g_quark_from_static_string (YOLO_SEGMENTATION_LOGITS);
163
164 gst_element_class_set_static_metadata (element_class,
165 "YOLO v8-11 segmentastion tensor decoder", "Tensordecoder/Video",
166 "Decode tensors output from the inference of Yolo or FastSAM model (segmentation)"
167 " on video frames. It works with YOLO version > 8 and FastSAM models.",
168 "Daniel Morin <[email protected]>, Santosh Mahto <[email protected]>");
169
170 gst_element_class_add_pad_template (element_class,
171 gst_static_pad_template_get (&gst_yolo_seg_tensor_decoder_sink_template));
172
173
174 basetransform_class->transform_ip = gst_yolo_seg_tensor_decoder_transform_ip;
175 basetransform_class->stop = gst_yolo_seg_tensor_decoder_stop;
176
177 od_class->object_found = gst_yolo_seg_tensor_decoder_object_found;
178
179 /* Workaround hotdoc bug */
180 gst_type_mark_as_plugin_api (GST_TYPE_YOLO_TENSOR_DECODER, 0);
181 }
182
183 static void
184 gst_yolo_seg_tensor_decoder_init (GstYoloSegTensorDecoder * self)
185 {
186 /* GstYoloSegTensorDecoder instance initialization */
187 self->mask_w = 0;
188 self->mask_h = 0;
189 self->mask_length = 0;
190 self->mask_pool = NULL;
191 memset (&self->mask_roi, 0, sizeof (BBox));
192
193 gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (self), FALSE);
194 }
195
196 static gboolean
197 gst_yolo_seg_tensor_decoder_get_tensors (GstYoloSegTensorDecoder * self,
198 GstBuffer * buf, const GstTensor ** logits_tensor,
199 const GstTensor ** detections_tensor)
200 {
201 GstMeta *meta = NULL;
202 gpointer iter_state = NULL;
203
204 if (!gst_buffer_get_meta (buf, GST_TENSOR_META_API_TYPE)) {
205 GST_WARNING_OBJECT (self,
206 "missing tensor meta from buffer %" GST_PTR_FORMAT, buf);
207 return FALSE;
208 }
209
210 while ((meta = gst_buffer_iterate_meta_filtered (buf, &iter_state,
211 GST_TENSOR_META_API_TYPE))) {
212 GstTensorMeta *tmeta = (GstTensorMeta *) meta;
213 const gsize YOLO_LOGITS_TENSOR_N_DIMS = 4;
214 static const gsize logits_dims[4] = { 1, G_MAXSIZE, G_MAXSIZE, G_MAXSIZE };
215 const gsize YOLO_DETECTIONS_TENSOR_N_DIMS = 3;
216 static const gsize detections_dims[3] = { 1, G_MAXSIZE, G_MAXSIZE };
217
218 *logits_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
219 YOLO_SEGMENTATION_LOGITS_TENSOR_ID, GST_TENSOR_DATA_TYPE_FLOAT32,
220 GST_TENSOR_DIM_ORDER_COL_MAJOR, YOLO_LOGITS_TENSOR_N_DIMS, logits_dims);
221 if (*logits_tensor == NULL)
222 continue;
223
224
225 *detections_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
226 YOLO_SEGMENTATION_DETECTION_MASK_ID, GST_TENSOR_DATA_TYPE_FLOAT32,
227 GST_TENSOR_DIM_ORDER_COL_MAJOR, YOLO_DETECTIONS_TENSOR_N_DIMS,
228 detections_dims);
229
230 if (*detections_tensor == NULL)
231 continue;
232
233 guint num_masks = (*logits_tensor)->dims[1];
234
235 if ((*detections_tensor)->dims[1] < 4 + 1 + num_masks) {
236 GST_WARNING_OBJECT (self, "Ignore tensor because dims[1] is %zu < %d",
237 (*detections_tensor)->dims[1], 4 + 1 + num_masks);
238 continue;
239 }
240
241 return TRUE;
242 }
243
244 return FALSE;
245 }
246
247 /* gst_yolo_seg_tensor_decoder_transform_ip:
248 * @trans: Instance
249 * @buf:inout: Buffer containing media and where tensors can be attached
250 * @return: Flow errors
251 * Decode Yolo tensors, post-process tensors and store decoded information
252 * into an analytics-meta that is attached to the buffer before been pushed
253 * downstream.
254 */
255 static GstFlowReturn
256 gst_yolo_seg_tensor_decoder_transform_ip (GstBaseTransform * trans,
257 GstBuffer * buf)
258 {
259 GstYoloSegTensorDecoder *self = GST_YOLO_SEG_TENSOR_DECODER (trans);
260 GstYoloTensorDecoder *od = GST_YOLO_TENSOR_DECODER (trans);
261 GstAnalyticsRelationMeta *rmeta;
262 gsize mask_w, mask_h;
263 const GstTensor *detections_tensor;
264 const GstTensor *logits_tensor;
265 GstFlowReturn ret = GST_FLOW_OK;
266 gboolean rv;
267
268 if (!gst_yolo_seg_tensor_decoder_get_tensors (self, buf, &logits_tensor,
269 &detections_tensor)) {
270 GST_WARNING_OBJECT (self,
271 "Couldn't find logit or detections tensor, skipping");
272 return GST_FLOW_OK;
273 }
274
275 rmeta = gst_buffer_add_analytics_relation_meta (buf);
276 if (rmeta == NULL) {
277 GST_ELEMENT_ERROR (trans, STREAM, FAILED, (NULL),
278 ("Analytics Relation meta allocation failed"));
279 return GST_FLOW_ERROR;
280 }
281
282 mask_w = logits_tensor->dims[2];
283 mask_h = logits_tensor->dims[3];
284
285 /* The detections need to be cropped to fit the SAR of the image. */
286 /* TODO: We're reconstructing the transformation that was done on the
287 * original image based on the assumption that the complete image without
288 * deformation would be analyzed. This assumption is not alway true and
289 * we should try to find a way to convey this transformation information
290 * and retrieve from here to know the transformation that need to be done
291 * on the mask.*/
292
293 if (self->mask_w != mask_w || self->mask_h != mask_h) {
294 self->mask_w = mask_w;
295 self->mask_h = mask_h;
296 self->mask_length = mask_w * mask_h;
297
298 if (od->video_info.width > od->video_info.height) {
299 self->bb2mask_gain = ((gfloat) self->mask_w) / od->video_info.width;
300 self->mask_roi.x = 0;
301 self->mask_roi.w = self->mask_w;
302 self->mask_roi.h = ((gfloat) self->bb2mask_gain) * od->video_info.height;
303 self->mask_roi.y = (self->mask_h - self->mask_roi.h) / 2;
304 } else {
305 self->bb2mask_gain = ((gfloat) self->mask_h) / od->video_info.height;
306 self->mask_roi.y = 0;
307 self->mask_roi.h = self->mask_h;
308 self->mask_roi.w = self->bb2mask_gain * od->video_info.width;
309 self->mask_roi.x = (self->mask_w - self->mask_roi.w) / 2;
310 }
311
312 if (self->mask_pool) {
313 gst_buffer_pool_set_active (self->mask_pool, FALSE);
314 g_clear_object (&self->mask_pool);
315 }
316 }
317
318 if (self->mask_pool == NULL) {
319 GstVideoInfo minfo;
320 GstCaps *caps;
321 gst_video_info_init (&minfo);
322 gst_video_info_set_format (&minfo, GST_VIDEO_FORMAT_GRAY8, self->mask_w,
323 self->mask_h);
324 caps = gst_video_info_to_caps (&minfo);;
325 self->mask_pool = gst_video_buffer_pool_new ();
326
327 GstStructure *config = gst_buffer_pool_get_config (self->mask_pool);
328 gst_buffer_pool_config_set_params (config, caps, self->mask_length, 0, 0);
329 gst_buffer_pool_config_add_option (config,
330 GST_BUFFER_POOL_OPTION_VIDEO_META);
331 gst_buffer_pool_set_config (self->mask_pool, config);
332 gst_buffer_pool_set_active (self->mask_pool, TRUE);
333 gst_caps_unref (caps);
334 }
335
336 /* Retrieve memory at index 0 from logits_tensor in READ mode */
337 rv = gst_buffer_map (logits_tensor->data, &self->map_info_logits,
338 GST_MAP_READ);
339 if (!rv) {
340 GST_ELEMENT_ERROR (self, STREAM, FAILED, (NULL),
341 ("Couldn't map logits tensor buffer: %" GST_PTR_FORMAT,
342 logits_tensor->data));
343 return GST_FLOW_ERROR;
344 }
345
346 self->logits_tensor = logits_tensor;
347
348 if (!gst_yolo_tensor_decoder_decode_f32 (od, rmeta, detections_tensor,
349 logits_tensor->dims[1]))
350 ret = GST_FLOW_ERROR;
351
352 gst_buffer_unmap (logits_tensor->data, &self->map_info_logits);
353
354 return ret;
355 }
356
357 static float
358 sigmoid (float x)
359 {
360 /* Check for positive overflow */
361 if (x > 0) {
362 double exp_neg_x = exp (-x);
363 return 1.0 / (1.0 + exp_neg_x);
364 }
365 /* Check for negative overflow and improve stability for negative x */
366 else {
367 double exp_x = exp (x);
368 return exp_x / (1.0 + exp_x);
369 }
370 }
371
372 static void
373 gst_yolo_seg_tensor_decoder_object_found (GstYoloTensorDecoder * od,
374 GstAnalyticsRelationMeta * rmeta, BBox * bb, gfloat confidence,
375 GQuark class_quark, const gfloat * candidate_masks, gsize offset,
376 guint count)
377 {
378 GstYoloSegTensorDecoder *self = GST_YOLO_SEG_TENSOR_DECODER (od);
379 GstAnalyticsODMtd od_mtd;
380 GstBuffer *mask_buf = NULL;
381 gfloat *data_logits = (gfloat *) self->map_info_logits.data;
382 BBox bb_mask;
383 GstFlowReturn flowret;
384 GstMapInfo out_mask_info;
385 guint region_ids[2] = { 0, count };
386 GstAnalyticsMtd seg_mtd;
387
388 gst_analytics_relation_meta_add_od_mtd (rmeta, class_quark,
389 bb->x, bb->y, bb->w, bb->h, confidence, &od_mtd);
390
391 bb_mask.x = self->bb2mask_gain * bb->x + self->mask_roi.x;
392 bb_mask.y = self->bb2mask_gain * bb->y + self->mask_roi.y;
393 bb_mask.w = self->bb2mask_gain * bb->w;
394 bb_mask.h = self->bb2mask_gain * bb->h;
395
396 flowret = gst_buffer_pool_acquire_buffer (self->mask_pool, &mask_buf, NULL);
397 g_assert (flowret == GST_FLOW_OK);
398 gst_buffer_map (mask_buf, &out_mask_info, GST_MAP_READWRITE);
399
400 GstVideoMeta *vmeta = gst_buffer_get_video_meta (mask_buf);
401 g_assert (vmeta != NULL);
402 vmeta->width = bb_mask.w;
403 vmeta->height = bb_mask.h;
404
405 #define MX_MAX (bb_mask.x + bb_mask.w)
406 #define MY_MAX (bb_mask.y + bb_mask.h)
407
408 for (gint my = bb_mask.y, i = 0; my < MY_MAX; my++) {
409 for (gint mx = bb_mask.x; mx < MX_MAX; mx++, i++) {
410 float sum = 0.0f;
411 gint j = my * self->mask_w + mx;
412 for (gsize k = 0; k < self->logits_tensor->dims[1]; ++k) {
413 GST_TRACE_OBJECT (self, "protos data at ((mx=%d,my=%d)=%d, %zu) is %f",
414 mx, my, j, k, data_logits[k * self->mask_length + j]);
415 sum += candidate_masks[offset * k] *
416 data_logits[k * self->mask_length + j];
417 }
418 out_mask_info.data[i] = sigmoid (sum) > 0.5 ? count : 0;
419 }
420 }
421
422 gst_analytics_relation_meta_add_segmentation_mtd (rmeta, mask_buf,
423 GST_SEGMENTATION_TYPE_INSTANCE, 1, region_ids, bb->x, bb->y, bb->w,
424 bb->h, &seg_mtd);
425
426 gst_analytics_relation_meta_set_relation (rmeta,
427 GST_ANALYTICS_REL_TYPE_RELATE_TO, od_mtd.id, seg_mtd.id);
428 gst_analytics_relation_meta_set_relation (rmeta,
429 GST_ANALYTICS_REL_TYPE_RELATE_TO, seg_mtd.id, od_mtd.id);
430
431
432 gst_buffer_unmap (mask_buf, &out_mask_info);
433 }