Debian Codegraph @

gst/tensordecoders/gstioutracker.c

raw
1 /*
2 * GStreamer gstreamer-ioutracker
3 * Copyright (C) 2025 Collabora Ltd.
4 * author: Santosh Mahto <[email protected]>
5 * gstioutracker.c
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
16 *
17 * You should have received a copy of the GNU Library General Public
18 * License along with this library; if not, write to the
19 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
20 * Boston, MA 02110-1301, USA.
21 */
22
23 /**
24 * SECTION:element-ioutracker
25 * @short_description: Simple object tracking based on Intersection-over-Union
26 *
27 * This element can parse per-buffer object-detection meta and add tracking information.
28 *
29 * It does an intersection-over-union, so it tracks if two detections share enough area to
30 * be likely to be the same thing.
31 *
32 * Note: This is meant for simplest cases of object tracking and has known limitations.
33 * For complex cases, please choose other advance tracking.
34 *
35 * \[
36 * gst-launch-1.0 filesrc location=bouncing.mp4 ! decodebin
37 * ! videoconvertscale add-borders=1 ! 'video/x-raw,pixel-aspect-ratio=1/1' \
38 * ! onnxinference execution-provider=cpu model-file=./yolov8s.onnx \
39 * ! yolotensordecoder class-confidence-threshold=0.5 \
40 * ! ioutracker iou-score-threshold=0.7 \
41 * ! videoconvert ! glimagesink
42 * ]|
43 *
44 * Since: 1.28
45 */
46
47 #ifdef HAVE_CONFIG_H
48 #include "config.h"
49 #endif
50
51 #include "gstioutracker.h"
52
53 #include <gst/gst.h>
54 #include <gst/video/video.h>
55 #include <gst/analytics/analytics.h>
56 #include <gst/analytics/gstanalytics_image_util.h>
57
58 GST_DEBUG_CATEGORY_STATIC (iou_tracker_debug);
59 #define GST_CAT_DEFAULT iou_tracker_debug
60 GST_ELEMENT_REGISTER_DEFINE (iou_tracker, "ioutracker",
61 GST_RANK_PRIMARY, GST_TYPE_IOU_TRACKER);
62
63 /* GstIouTracker properties */
64 enum
65 {
66 PROP_0,
67 PROP_IOU_SCORE_THRESHOLD,
68 PROP_MIN_FRAME_COUNT_FOR_LOST_TRACK
69 };
70
71 #define DEFAULT_MIN_FRAME_COUNT_FOR_LOST_TRACK 5 /* randomly chosen */
72 #define DEFAULT_IOU_SCORE_THRESHOLD 0.5f /* 0 to 1 */
73
74 static GstStaticPadTemplate gst_iou_tracker_src_template =
75 GST_STATIC_PAD_TEMPLATE ("src",
76 GST_PAD_SRC,
77 GST_PAD_ALWAYS,
78 GST_STATIC_CAPS ("video/x-raw")
79 );
80
81 static GstStaticPadTemplate gst_iou_tracker_sink_template =
82 GST_STATIC_PAD_TEMPLATE ("sink",
83 GST_PAD_SINK,
84 GST_PAD_ALWAYS,
85 GST_STATIC_CAPS ("video/x-raw")
86 );
87
88 typedef struct _BBox
89 {
90 gint x;
91 gint y;
92 gint w;
93 gint h;
94 } BBox;
95
96 typedef struct _TrackData
97 {
98 guint64 id;
99 GstClockTime first_seen; // First time object was seen
100 GstClockTime last_seen; // Last time object was seen
101 GstClockTime last_tracked; // Last time object was tracked
102 guint unseen_frame_count; // Last frame seen
103 gboolean lost; // Whether the object is lost or not
104 GQuark obj_type; // The object type from the object detection
105 GQueue bbqueue; // List of bounding boxes history for the object
106 } TrackData;
107
108 static void gst_iou_tracker_set_property (GObject * object,
109 guint prop_id, const GValue * value, GParamSpec * pspec);
110 static void gst_iou_tracker_get_property (GObject * object,
111 guint prop_id, GValue * value, GParamSpec * pspec);
112 static void gst_iou_tracker_finalize (GObject * object);
113 static GstFlowReturn gst_iou_tracker_transform_ip (GstBaseTransform *
114 trans, GstBuffer * buf);
115 static gboolean gst_iou_tracker_stop (GstBaseTransform * trans);
116
117 G_DEFINE_TYPE (GstIouTracker, gst_iou_tracker, GST_TYPE_BASE_TRANSFORM);
118
119 static void
120 gst_iou_tracker_class_init (GstIouTrackerClass * klass)
121 {
122 GObjectClass *gobject_class = (GObjectClass *) klass;
123 GstElementClass *element_class = (GstElementClass *) klass;
124 GstBaseTransformClass *basetransform_class = (GstBaseTransformClass *) klass;
125
126 GST_DEBUG_CATEGORY_INIT (iou_tracker_debug, "ioutracker", 0,
127 "Intersection-over-Union tracker");
128
129 gobject_class->set_property = gst_iou_tracker_set_property;
130 gobject_class->get_property = gst_iou_tracker_get_property;
131 gobject_class->finalize = gst_iou_tracker_finalize;
132
133 /**
134 * GstIouTracker:iou-score-threshold
135 *
136 * The score below which object is considered as different object.
137 *
138 */
139 g_object_class_install_property (G_OBJECT_CLASS (klass),
140 PROP_IOU_SCORE_THRESHOLD, g_param_spec_float ("iou-score-threshold",
141 "IoU Score threshold",
142 "Threshold for deciding wether the object is same in different frames",
143 0.0, 1.0, DEFAULT_IOU_SCORE_THRESHOLD,
144 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
145
146 /**
147 * GstIouTracker:min-frame-count-for-lost-track
148 *
149 * Min number of frame where object is not seen, required to mark object as lost.
150 *
151 */
152 g_object_class_install_property (G_OBJECT_CLASS (klass),
153 PROP_MIN_FRAME_COUNT_FOR_LOST_TRACK,
154 g_param_spec_uint ("min-frame-count-for-lost-track",
155 "Min consecutive frame count for lost track",
156 "Min number of consecutive frames where object is absent before track is considered lost",
157 0, G_MAXUINT, DEFAULT_MIN_FRAME_COUNT_FOR_LOST_TRACK,
158 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
159
160 gst_element_class_set_static_metadata (element_class,
161 "Intersection-over-Union (IoU) object tracker", "Analyzer/Video",
162 "Track the objects across frames based on Intersection-over-Union (IoU)",
163 "Santosh Mahto <[email protected]>");
164 gst_element_class_add_pad_template (element_class,
165 gst_static_pad_template_get (&gst_iou_tracker_sink_template));
166 gst_element_class_add_pad_template (element_class,
167 gst_static_pad_template_get (&gst_iou_tracker_src_template));
168
169 basetransform_class->transform_ip =
170 GST_DEBUG_FUNCPTR (gst_iou_tracker_transform_ip);
171 basetransform_class->stop = gst_iou_tracker_stop;
172 }
173
174 static void
175 gst_iou_tracker_track_data_free (TrackData * data)
176 {
177 if (!data)
178 return;
179
180 g_queue_clear_full (&data->bbqueue, (GDestroyNotify) g_free);
181 g_free (data);
182 }
183
184 static gboolean
185 gst_iou_tracker_stop (GstBaseTransform * trans)
186 {
187 GstIouTracker *self = GST_IOU_TRACKER (trans);
188 g_hash_table_remove_all (self->picked_odmtds);
189 g_list_free_full (self->tracks,
190 (GDestroyNotify) gst_iou_tracker_track_data_free);
191 self->tracks = NULL;
192
193 return TRUE;
194 }
195
196 static void
197 gst_iou_tracker_init (GstIouTracker * self)
198 {
199 self->min_frame_count_for_lost_track = DEFAULT_MIN_FRAME_COUNT_FOR_LOST_TRACK;
200 self->iou_score_threshold = DEFAULT_IOU_SCORE_THRESHOLD;
201 self->tracks = NULL;
202 self->next_track_id = 0;
203
204 self->picked_odmtds = g_hash_table_new_full (g_direct_hash,
205 g_direct_equal, NULL, NULL);
206
207 gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (self), FALSE);
208 }
209
210 static void
211 gst_iou_tracker_finalize (GObject * object)
212 {
213 GstIouTracker *self = GST_IOU_TRACKER (object);
214 g_hash_table_destroy (self->picked_odmtds);
215 g_list_free_full (self->tracks,
216 (GDestroyNotify) gst_iou_tracker_track_data_free);
217 G_OBJECT_CLASS (gst_iou_tracker_parent_class)->finalize (object);
218 }
219
220 static void
221 gst_iou_tracker_set_property (GObject * object, guint prop_id,
222 const GValue * value, GParamSpec * pspec)
223 {
224 GstIouTracker *self = GST_IOU_TRACKER (object);
225
226 switch (prop_id) {
227 case PROP_IOU_SCORE_THRESHOLD:
228 self->iou_score_threshold = g_value_get_float (value);
229 break;
230 case PROP_MIN_FRAME_COUNT_FOR_LOST_TRACK:
231 self->min_frame_count_for_lost_track = g_value_get_uint (value);
232 break;
233 default:
234 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
235 break;
236 }
237 }
238
239 static void
240 gst_iou_tracker_get_property (GObject * object, guint prop_id,
241 GValue * value, GParamSpec * pspec)
242 {
243 GstIouTracker *self = GST_IOU_TRACKER (object);
244
245 switch (prop_id) {
246 case PROP_IOU_SCORE_THRESHOLD:
247 g_value_set_float (value, self->iou_score_threshold);
248 break;
249 case PROP_MIN_FRAME_COUNT_FOR_LOST_TRACK:
250 g_value_set_uint (value, self->min_frame_count_for_lost_track);
251 break;
252 default:
253 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
254 break;
255 }
256 }
257
258 static gfloat
259 gst_iou_tracker_get_iou (BBox b1, BBox b2)
260 {
261 return gst_analytics_image_util_iou_float (b1.x, b1.y, b1.w, b1.h,
262 b2.x, b2.y, b2.w, b2.h);
263 }
264
265 static GstFlowReturn
266 gst_iou_tracker_transform_ip (GstBaseTransform * trans, GstBuffer * buf)
267 {
268 GstIouTracker *self = GST_IOU_TRACKER (trans);
269 GstAnalyticsRelationMeta *rmeta;
270 GstAnalyticsMtd mtd;
271 GstClockTime pt = GST_BUFFER_PTS (buf);
272 GstClockTime running_time;
273 gpointer state = NULL;
274
275 rmeta = gst_buffer_get_analytics_relation_meta (buf);
276
277 if (!rmeta) {
278 GST_DEBUG_OBJECT (self, "No GstAnalyticsRelationMeta found in buffer");
279 if (self->tracks) {
280 // Tracking has started, so add an rmeta to allow adding
281 // TrackingMtd.
282 rmeta = gst_buffer_add_analytics_relation_meta (buf);
283 } else {
284 return GST_FLOW_OK;
285 }
286 }
287
288 g_hash_table_remove_all (self->picked_odmtds);
289 running_time =
290 gst_segment_to_running_time (&trans->segment, GST_FORMAT_TIME, pt);
291
292 /*
293 * Interate over all the existing tracks and update them with new detections.
294 * When object is not seen in `min_lost_frame_count_to_remove_track` consecutive frames,
295 * the mark it as lost and track are removed, until then track the object and assume
296 * predicted position as new position.
297 */
298 GList *track = self->tracks;
299 while (track) {
300 TrackData *tdata = (TrackData *) track->data;
301 GstAnalyticsODMtd nearest_mtd;
302 GstAnalyticsTrackingMtd tmtd;
303 gfloat max_iou_score = 0.0f;
304 gpointer state = NULL;
305 BBox *cbox;
306
307 // Add the older tracking meta to the relation meta to this new buffer
308 if (!gst_analytics_relation_meta_add_tracking_mtd (rmeta,
309 tdata->id, tdata->first_seen, &tmtd)) {
310 GST_DEBUG_OBJECT (self, "Failed to add tracking meta to relation meta");
311 continue;
312 }
313
314 gst_analytics_tracking_mtd_update_last_seen (&tmtd, tdata->last_seen);
315 if (tdata->lost) {
316 gst_analytics_tracking_mtd_set_lost (&tmtd);
317 }
318
319 cbox = g_queue_peek_head (&tdata->bbqueue);
320 // Iterate over od mtds in current frame and find the latest position of
321 // tracked object based on IOU score.
322 while (gst_analytics_relation_meta_iterate (rmeta, &state,
323 gst_analytics_od_mtd_get_mtd_type (), &mtd)) {
324 GstAnalyticsODMtd *od_mtd = (GstAnalyticsODMtd *) & mtd;
325 BBox odbox;
326 gfloat iou_score = -0.0f;
327
328 if (g_hash_table_contains (self->picked_odmtds,
329 GINT_TO_POINTER (od_mtd->id)))
330 continue;
331
332 /* Different type, ignore it */
333 if (tdata->obj_type != gst_analytics_od_mtd_get_obj_type (od_mtd))
334 continue;
335
336 gst_analytics_od_mtd_get_location (od_mtd, &odbox.x, &odbox.y,
337 &odbox.w, &odbox.h, NULL);
338
339 // Note: IoU based tracking fails when object position doesn't overlap across frame since iou
340 // becomes zero. This mostly happens when frame rate are low or object is moving fast.
341 // This is known limitation of current implementation.
342 iou_score = gst_iou_tracker_get_iou (odbox, *cbox);
343 if (iou_score > max_iou_score) {
344 max_iou_score = iou_score;
345 nearest_mtd = *od_mtd;
346 }
347 }
348
349 if (max_iou_score >= self->iou_score_threshold) {
350 BBox *new_box = g_new0 (BBox, 1);
351
352 gst_analytics_od_mtd_get_location (&nearest_mtd, &new_box->x,
353 &new_box->y, &new_box->w, &new_box->h, NULL);
354
355 g_queue_push_head (&tdata->bbqueue, new_box);
356
357 tdata->last_seen = running_time;
358 tdata->last_tracked = running_time;
359 tdata->unseen_frame_count = 0;
360 gst_analytics_tracking_mtd_update_last_seen (&tmtd, running_time);
361 gst_analytics_relation_meta_set_relation (rmeta,
362 GST_ANALYTICS_REL_TYPE_RELATE_TO, nearest_mtd.id, tmtd.id);
363
364 GST_DEBUG_OBJECT (self,
365 "Total track: %u, Track %" G_GUINT64_FORMAT
366 " updated with new last seen time: %"
367 GST_TIME_FORMAT, g_list_length (self->tracks),
368 tdata->id, GST_TIME_ARGS (tdata->last_seen));
369
370 g_hash_table_insert (self->picked_odmtds,
371 GINT_TO_POINTER (nearest_mtd.id), (gpointer) TRUE);
372 } else {
373 tdata->unseen_frame_count++;
374
375 // Remove the track once we have seen enough frame where object was missing.
376 if (tdata->unseen_frame_count >= self->min_frame_count_for_lost_track) {
377 gst_analytics_tracking_mtd_set_lost (&tmtd);
378 GST_DEBUG_OBJECT (self, "Track %" G_GUINT64_FORMAT " marked as lost",
379 tdata->id);
380 GList *nexttrack = track->next;
381 guint64 trackid = tdata->id; // Logging purpose
382 // Remove current track from the list
383 // caution: list element is freed within iteration
384 gst_iou_tracker_track_data_free (tdata);
385 self->tracks = g_list_delete_link (self->tracks, track);
386 track = nexttrack;
387
388 GST_DEBUG_OBJECT (self,
389 "Track %" G_GUINT64_FORMAT " FORMAT after %u unseen frames",
390 trackid, tdata->unseen_frame_count);
391
392 continue; // start next iteration
393 } else {
394 // Since object is not seen in this frame, we need to calulate predicted position
395 // based on previous position change
396 guint count = tdata->bbqueue.length;
397 BBox *new_box = g_new0 (BBox, 1);
398
399 BBox *cur_box = g_queue_peek_head (&tdata->bbqueue);
400 BBox *last_box = g_queue_peek_tail (&tdata->bbqueue);
401
402 new_box->x = cur_box->x + (cur_box->x - last_box->x) / count;
403 new_box->y = cur_box->y + (cur_box->y - last_box->y) / count;
404 new_box->w = cur_box->w;
405 new_box->h = cur_box->h;
406 g_queue_push_head (&tdata->bbqueue, new_box);
407 tdata->last_tracked = running_time;
408 GST_DEBUG_OBJECT (self, "Track %" G_GUINT64_FORMAT
409 " not updated, but predicted position is (%d, %d, %d, %d)",
410 tdata->id, new_box->x, new_box->y, new_box->w, new_box->h);
411 }
412 }
413 track = track->next;
414 }
415
416 // Add new tracks for all the new object found in detection. so for the first time
417 // tracks for all the detections are created.
418 while (gst_analytics_relation_meta_iterate (rmeta, &state,
419 gst_analytics_od_mtd_get_mtd_type (), &mtd)) {
420 GstAnalyticsODMtd *od_mtd = (GstAnalyticsODMtd *) & mtd;
421 GstAnalyticsTrackingMtd tmtd;
422
423 if (!g_hash_table_contains (self->picked_odmtds,
424 GINT_TO_POINTER (od_mtd->id))) {
425 // If the mtd is not picked, it means it is not matched with any track
426 // and hence it is a new detection
427 if (!gst_analytics_relation_meta_add_tracking_mtd (rmeta,
428 self->next_track_id, running_time, &tmtd)) {
429 GST_ELEMENT_ERROR (self, STREAM, FAILED, (NULL),
430 ("Failed to add tracking mtd for new track"));
431 return GST_FLOW_ERROR;
432 }
433
434 BBox *new_bbox = g_new0 (BBox, 1);
435 gst_analytics_od_mtd_get_location (od_mtd, &new_bbox->x, &new_bbox->y,
436 &new_bbox->w, &new_bbox->h, NULL);
437
438 TrackData *new_track_data = g_new0 (TrackData, 1);
439 g_queue_init (&new_track_data->bbqueue);
440 g_queue_push_head (&new_track_data->bbqueue, new_bbox);
441 new_track_data->id = self->next_track_id;
442 new_track_data->first_seen = running_time;
443 new_track_data->last_seen = running_time;
444 new_track_data->last_tracked = running_time;
445 new_track_data->lost = FALSE;
446 new_track_data->unseen_frame_count = 0;
447 new_track_data->obj_type = gst_analytics_od_mtd_get_obj_type (od_mtd);
448 self->tracks = g_list_append (self->tracks, new_track_data);
449 GST_DEBUG_OBJECT (self,
450 "New track created with ID: %" G_GUINT64_FORMAT
451 ", First Seen: %" GST_TIME_FORMAT,
452 new_track_data->id, GST_TIME_ARGS (new_track_data->first_seen));
453
454 self->next_track_id++;
455
456 if (!gst_analytics_relation_meta_set_relation (rmeta,
457 GST_ANALYTICS_REL_TYPE_RELATE_TO, od_mtd->id, tmtd.id)) {
458 GST_ERROR_OBJECT (self,
459 "Failed to set relation for new track Tracking ID: %u and ODM ID: %u",
460 self->next_track_id, od_mtd->id);
461 }
462 }
463 }
464
465 // picked_odmtds is used to keep track of the ODMs for single buffer only, so free it.
466 g_hash_table_remove_all (self->picked_odmtds);
467
468 return GST_FLOW_OK;
469 }