Debian Codegraph @

core/utilities/facemanagement/pipelines/edit/facepipelineedit.cpp

raw
1 /* ============================================================
2 *
3 * This file is a part of digiKam project
4 * https://www.digikam.org
5 *
6 * Date : 2024-11-10
7 * Description : Performs face detection and recognition
8 *
9 * SPDX-FileCopyrightText: 2024-2026 by Gilles Caulier <caulier dot gilles at gmail dot com>
10 * SPDX-FileCopyrightText: 2024-2025 by Michael Miller <michael underscore miller at msn dot com>
11 *
12 * SPDX-License-Identifier: GPL-2.0-or-later
13 *
14 * ============================================================ */
15
16 #include "facepipelineedit.h"
17
18 // Qt includes
19
20 #include <QList>
21 #include <QSet>
22 #include <QElapsedTimer>
23 #include <QRectF>
24
25 // KDE includes
26
27 #include <klocalizedstring.h>
28
29 // Local includes
30
31 #include "digikam_debug.h"
32 #include "digikam_opencv.h"
33 #include "sharedqueue.h"
34 #include "collectionmanager.h"
35 #include "album.h"
36 #include "iteminfo.h"
37 #include "coredb.h"
38 #include "facescansettings.h"
39 #include "dimg.h"
40 #include "facepipelinepackagebase.h"
41 #include "identityprovider.h"
42 #include "identity.h"
43 #include "dnnsfaceextractor.h"
44 #include "metadatahub.h"
45 #include "scancontroller.h"
46 #include "facedetector.h"
47 #include "faceclassifier.h"
48
49 namespace Digikam
50 {
51
52 class Q_DECL_HIDDEN FacePipelineEditCreator
53 {
54 public:
55
56 FacePipelineEdit object;
57 };
58
59 Q_GLOBAL_STATIC(FacePipelineEditCreator, creator)
60
61 // ----------------------------------------------
62
63 FacePipelineEdit::FacePipelineEdit()
64 : FacePipelineBase(FaceScanSettings())
65 {
66 debugConfirmTimer.start();
67 }
68
69 FacePipelineEdit* FacePipelineEdit::instance()
70 {
71 return &creator->object;
72 }
73
74 FaceTagsIface FacePipelineEdit::confirmFace(const ItemInfo& info,
75 const FaceTagsIface& face,
76 const TagRegion& region,
77 int tagId,
78 bool retrain)
79 {
80 if (debugConfirmTimer.elapsed() < 250)
81 {
82 qCDebug(DIGIKAM_FACESENGINE_LOG) << "FacePipelineEdit::confirmFace(): INFO: more than 1 "
83 "face confirmed in less than 0.25 seconds";
84 }
85
86 debugConfirmTimer.restart();
87
88 FaceTagsIface newFace = getRejectedFaceTagList(face);
89
90 // CR #1680458 optimization: Check if the queue exists before allocating 'package'.
91 // This avoids unnecessary memory allocation and eliminates the need
92 // for manual 'delete' in error cases, preventing potential memory leaks.
93 // The 'package' object is only created if the queue is available.
94
95 if (!queues.contains(MLPipelineStage::Writer))
96 {
97 qCWarning(DIGIKAM_FACESENGINE_LOG) << "No next queue found for MLPipelineStage::Writer";
98
99 return (FaceTagsEditor::confirmedEntry(newFace, tagId, region));
100 }
101
102 MLPipelineQueue* const nextQueue = queues.value(MLPipelineStage::Loader);
103 FacePipelinePackageBase* const package = new FacePipelinePackageBase(info,
104 newFace,
105 tagId,
106 region,
107 DImg(),
108 FacePipelinePackageBase::EditPipelineAction::Confirm,
109 retrain);
110
111 ++totalItemCount;
112 Q_EMIT scheduled();
113 Q_EMIT started(i18n("Confirming face"));
114
115 emitSignalUpdateItemCount(nextQueue->size()+1);
116
117 enqueue(nextQueue, package);
118
119 return (FaceTagsEditor::confirmedEntry(newFace, tagId, region));
120 }
121
122 void FacePipelineEdit::removeFace(const ItemInfo& info,
123 const FaceTagsIface& face)
124 {
125 FaceTagsIface newFace = getRejectedFaceTagList(face);
126
127 // Optimization: Check if the queue exists before allocating 'package'.
128 // This avoids unnecessary memory allocation and eliminates the need
129 // for manual 'delete' in error cases, preventing potential memory leaks.
130 // The 'package' object is only created if the queue is available.
131
132 if (!queues.contains(MLPipelineStage::Writer))
133 {
134 qCWarning(DIGIKAM_FACESENGINE_LOG) << "No next queue found for MLPipelineStage::Writer";
135
136 return;
137 }
138
139 MLPipelineQueue* const nextQueue = queues.value(MLPipelineStage::Writer);
140 FacePipelinePackageBase* const package = new FacePipelinePackageBase(info,
141 newFace,
142 newFace.tagId(),
143 newFace.region(),
144 DImg(),
145 FacePipelinePackageBase::EditPipelineAction::Remove,
146 newFace.isConfirmedName());
147
148 ++totalItemCount;
149
150 Q_EMIT scheduled();
151 Q_EMIT started(i18n("Removing face"));
152
153 emitSignalUpdateItemCount(nextQueue->size()+1);
154
155 enqueue(nextQueue, package);
156 }
157
158 void FacePipelineEdit::removeAllFaces(const ItemInfo& info)
159 {
160 // Optimization: Check if the queue exists before allocating 'package'.
161 // This avoids unnecessary memory allocation and eliminates the need
162 // for manual 'delete' in error cases, preventing potential memory leaks.
163 // The 'package' object is only created if the queue is available.
164
165 if (!queues.contains(MLPipelineStage::Writer))
166 {
167 qCWarning(DIGIKAM_FACESENGINE_LOG) << "No next queue found for MLPipelineStage::Writer";
168
169 return;
170 }
171
172 MLPipelineQueue* const nextQueue = queues.value(MLPipelineStage::Writer);
173 FacePipelinePackageBase* const package = new FacePipelinePackageBase(info,
174 FacePipelinePackageBase::EditPipelineAction::RemoveAll);
175
176 ++totalItemCount;
177
178 Q_EMIT scheduled();
179 Q_EMIT started(i18n("Remove all faces"));
180
181 emitSignalUpdateItemCount(nextQueue->size()+1);
182
183 enqueue(nextQueue, package);
184 }
185
186 FaceTagsIface FacePipelineEdit::editTag(const ItemInfo& info,
187 const FaceTagsIface& face,
188 int newTagId)
189 {
190 FaceTagsIface newFace = getRejectedFaceTagList(face);
191
192 // Optimization: Check if the queue exists before allocating 'package'.
193 // This avoids unnecessary memory allocation and eliminates the need
194 // for manual 'delete' in error cases, preventing potential memory leaks.
195 // The 'package' object is only created if the queue is available.
196
197 if (!queues.contains(MLPipelineStage::Writer))
198 {
199 qCWarning(DIGIKAM_FACESENGINE_LOG) << "No next queue found for MLPipelineStage::Writer";
200
201 return newFace;
202 }
203
204 MLPipelineQueue* const nextQueue = queues.value(MLPipelineStage::Writer);
205 FacePipelinePackageBase* const package = new FacePipelinePackageBase(info,
206 newFace,
207 newTagId,
208 newFace.region(),
209 DImg(),
210 FacePipelinePackageBase::EditPipelineAction::EditTag,
211 face.isConfirmedName());
212
213 ++totalItemCount;
214
215 Q_EMIT scheduled();
216 Q_EMIT started(i18n("Editing face tag"));
217
218 emitSignalUpdateItemCount(nextQueue->size()+1);
219
220 enqueue(nextQueue, package);
221
222 // FaceTagsIface newFace(package->face);
223 newFace.setTagId(newTagId);
224
225 return newFace;
226 }
227
228 FaceTagsIface FacePipelineEdit::editRegion(const ItemInfo& info,
229 const FaceTagsIface& face,
230 const TagRegion& region,
231 int tagId,
232 bool retrain)
233 {
234 FaceTagsIface newFace = getRejectedFaceTagList(face);
235
236 // Optimization: Check if the queue exists before allocating 'package'.
237 // This avoids unnecessary memory allocation and eliminates the need
238 // for manual 'delete' in error cases, preventing potential memory leaks.
239 // The 'package' object is only created if the queue is available.
240
241 if (!queues.contains(MLPipelineStage::Writer))
242 {
243 qCWarning(DIGIKAM_FACESENGINE_LOG) << "No next queue found for MLPipelineStage::Writer";
244
245 return newFace;
246 }
247
248 MLPipelineQueue* const nextQueue = queues.value(MLPipelineStage::Writer);
249 FacePipelinePackageBase* const package = new FacePipelinePackageBase(info,
250 newFace,
251 tagId,
252 region,
253 DImg(),
254 FacePipelinePackageBase::EditPipelineAction::EditRegion,
255 retrain);
256
257 ++totalItemCount;
258
259 Q_EMIT scheduled();
260 Q_EMIT started(i18n("Editing face region"));
261
262 emitSignalUpdateItemCount(nextQueue->size()+1);
263
264 enqueue(nextQueue, package);
265
266 newFace.setRegion(region);
267
268 if (tagId != FaceClassifierBase::UNKNOWN_LABEL_ID)
269 {
270 newFace.setTagId(tagId);
271 newFace.setType(FaceTagsIface::typeForId(tagId));
272 }
273
274 return newFace;
275 }
276
277 FaceTagsIface FacePipelineEdit::addManually(const ItemInfo& info,
278 const DImg& image,
279 const TagRegion& region,
280 bool retrain)
281 {
282 FaceTagsIface newFace = FaceTagsEditor::unconfirmedEntry(info.id(),
283 FaceClassifier::UNKNOWN_LABEL_ID,
284 region, QList<int>());
285
286 // Optimization: Check if the queue exists before allocating 'package'.
287 // This avoids unnecessary memory allocation and eliminates the need
288 // for manual 'delete' in error cases, preventing potential memory leaks.
289 // The 'package' object is only created if the queue is available.
290
291 if (!queues.contains(MLPipelineStage::Writer))
292 {
293 qCWarning(DIGIKAM_FACESENGINE_LOG) << "No next queue found for MLPipelineStage::Writer";
294
295 return newFace;
296 }
297
298 MLPipelineQueue* const nextQueue = queues.value(MLPipelineStage::Writer);
299 FacePipelinePackageBase* const package = new FacePipelinePackageBase(info,
300 newFace,
301 newFace.tagId(),
302 newFace.region(),
303 image,
304 FacePipelinePackageBase::EditPipelineAction::AddManually,
305 retrain);
306
307 ++totalItemCount;
308
309 Q_EMIT scheduled();
310 Q_EMIT started(i18n("Manually adding face"));
311
312 emitSignalUpdateItemCount(nextQueue->size()+1);
313
314 enqueue(nextQueue, package);
315
316 return newFace;
317 }
318
319 QList<FaceTagsIface> FacePipelineEdit::deleteRejectedFaceTagLists(const ItemInfo& info)
320 {
321 QList<FaceTagsIface> result;
322 QList<FaceTagsIface> faces = FaceUtils().databaseFaces(info.id());
323
324 for (const FaceTagsIface& face : std::as_const(faces))
325 {
326 result << deleteRejectedFaceTagList(face);
327 }
328
329 return result;
330 }
331
332 FaceTagsIface FacePipelineEdit::deleteRejectedFaceTagList(const FaceTagsIface& face)
333 {
334 FaceTagsIface newFace(face);
335 newFace.clearRejectedFaceTagList();
336
337 // Optimization: Check if the queue exists before allocating 'package'.
338 // This avoids unnecessary memory allocation and eliminates the need
339 // for manual 'delete' in error cases, preventing potential memory leaks.
340 // The 'package' object is only created if the queue is available.
341
342 if (!queues.contains(MLPipelineStage::Writer))
343 {
344 qCWarning(DIGIKAM_FACESENGINE_LOG) << "No next queue found for MLPipelineStage::Writer";
345
346 return newFace;
347 }
348
349 MLPipelineQueue* const nextQueue = queues.value(MLPipelineStage::Writer);
350 FacePipelinePackageBase* const package = new FacePipelinePackageBase(ItemInfo(face.imageId()),
351 face,
352 face.tagId(),
353 face.region(),
354 DImg(),
355 FacePipelinePackageBase::EditPipelineAction::DeleteRejectedFaceTagList,
356 false);
357
358 ++totalItemCount;
359
360 Q_EMIT scheduled();
361 Q_EMIT started(i18n("Clearing rejected face tag list."));
362
363 emitSignalUpdateItemCount(nextQueue->size()+1);
364
365 enqueue(nextQueue, package);
366
367 return newFace;
368 }
369
370 bool FacePipelineEdit::start()
371 {
372 if (!isStarted)
373 {
374 isStarted = true;
375
376 {
377 // use the mutex to synchronize the start of the threads
378
379 QMutexLocker lock(&mutex);
380
381 // add the worker threads for this pipeline
382 /*
383 addWorker(MLPipelineStage::Finder);
384 */
385 addWorker(MLPipelineStage::Loader);
386 addWorker(MLPipelineStage::Extractor);
387 /*
388 addWorker(MLPipelineStage::Classifier);
389 */
390 addWorker(MLPipelineStage::Writer);
391 }
392
393 return FacePipelineBase::start();
394 }
395 else
396 {
397 return true;
398 }
399 }
400
401 bool FacePipelineEdit::loader()
402 {
403 ActionThreadBase::setCurrentThreadName(QString::fromLatin1("FaceEdit%1").arg(QLatin1String(__FUNCTION__))); // To customize thread name
404
405 return commonFaceThumbnailLoader(QStringLiteral("FacePipelineEdit"),
406 QThread::NormalPriority,
407 MLPipelineStage::Loader,
408 MLPipelineStage::Extractor);
409 }
410
411 bool FacePipelineEdit::extractor()
412 {
413 ActionThreadBase::setCurrentThreadName(QString::fromLatin1("FaceEdit%1").arg(QLatin1String(__FUNCTION__))); // To customize thread name
414
415 return commonFaceThumbnailExtractor(QStringLiteral("FacePipelineEdit"),
416 QThread::NormalPriority,
417 MLPipelineStage::Extractor,
418 MLPipelineStage::Writer,
419 true);
420 }
421
422 bool FacePipelineEdit::writer()
423 {
424 ActionThreadBase::setCurrentThreadName(QString::fromLatin1("FaceEdit%1").arg(QLatin1String(__FUNCTION__))); // To customize thread name
425
426 MLPIPELINE_STAGE_START(QThread::NormalPriority, MLPipelineStage::Writer, MLPipelineStage::None);
427 FacePipelinePackageBase* package = nullptr;
428
429 /* =========================================================================================
430 * Pipeline stage specific initialization code
431 *
432 * Use the block from here to MLPIPELINE_LOOP_START to initialize the stage.
433 * The code in this block is run once per stage initialization. The number of instances
434 * is at least 1. More instances are created by addMoreWorkers if needed.
435 */
436
437 //--------------------------------------------------------------------------------
438
439 IdentityProvider* const idProvider = IdentityProvider::instance();
440
441 // override the default queue depth
442
443 thisQueue->setMaxDepth(thisQueue->maxDepthLimit());
444
445 MLPIPELINE_LOOP_START(MLPipelineStage::Writer, thisQueue);
446 package = static_cast<FacePipelinePackageBase*>(mlpackage);
447
448 /* =========================================================================================
449 * Start pipeline stage specific loop
450 *
451 * All code from here to MLPIPELINE_LOOP_END is in a try/catch block and loop.
452 * This loop is run once per image.
453 */
454
455 {
456 FaceUtils utils;
457
458 switch (package->action)
459 {
460 case FacePipelinePackageBase::EditPipelineAction::Confirm:
461 {
462 TagRegion confirmedRegion = package->region.isValid() ? package->region
463 : package->face.region();
464 FaceTagsIface confirmedFace = utils.confirmName(package->face, package->tagId, confirmedRegion);
465 Identity identity = utils.identityForTag(confirmedFace.tagId());
466
467 if (0 != package->features.rows)
468 {
469 if (package->useForTraining)
470 {
471 idProvider->addTraining(identity, confirmedFace.hash(), package->features);
472 }
473 else
474 {
475 qCDebug(DIGIKAM_FACESENGINE_LOG) << "FacePipelineEdit::writer(): not using for training: "
476 << package->info.filePath();
477 }
478 }
479 else
480 {
481 qCDebug(DIGIKAM_FACESENGINE_LOG) << "FacePipelineEdit::writer(): bad mat";
482 }
483
484 break;
485 }
486
487 case FacePipelinePackageBase::EditPipelineAction::Remove:
488 {
489 utils.removeFace(package->face);
490 break;
491 }
492
493 case FacePipelinePackageBase::EditPipelineAction::RemoveAll:
494 {
495 utils.removeAllFaces(package->info.id());
496 break;
497 }
498
499 case FacePipelinePackageBase::EditPipelineAction::EditTag:
500 {
501 // Change Tag operation.
502
503 if (
504 (package->face.isUnconfirmedName()) &&
505 (FaceTags::unknownPersonTagId() != package->face.tagId()) &&
506 (FaceTags::unknownPersonTagId() == package->tagId)
507 )
508 {
509 // The face is unconfirmed and the tag is not the unknown person tag.
510
511 utils.rejectSuggestedTag(package->face);
512 }
513 else
514 {
515 utils.changeTag(package->face, package->tagId);
516 }
517
518 break;
519 }
520
521 case FacePipelinePackageBase::EditPipelineAction::EditRegion:
522 {
523 if (package->face.region() != package->region)
524 {
525 package->face = utils.changeRegion(package->face, package->region);
526 }
527
528 if (package->tagId != FaceClassifier::UNKNOWN_LABEL_ID)
529 {
530 utils.changeTag(package->face, package->tagId);
531 }
532
533 break;
534 }
535
536 case FacePipelinePackageBase::EditPipelineAction::AddManually:
537 {
538 utils.addManually(utils.unconfirmedEntry(package->info.id(), package->tagId,
539 package->region, package->face.rejectedFaceTagList()));
540 break;
541 }
542
543 case FacePipelinePackageBase::EditPipelineAction::DeleteRejectedFaceTagList:
544 {
545 utils.removeRejectedFaceTagList(package->face);
546 break;
547 }
548 }
549
550 // update the tags
551
552 if (utils.normalTagChanged())
553 {
554 MetadataHub hub;
555 hub.load(package->info);
556
557 ScanController::FileMetadataWrite writeScope(package->info);
558 writeScope.changed(hub.writeToMetadata(package->info, MetadataHub::WRITE_TAGS));
559 }
560
561 // retrain the face classifier if the retrain flag is set
562
563 if (package->retrain)
564 {
565 FaceClassifier::instance()->retrain();
566 }
567
568 // send a notification that the image was processed
569
570 notify(MLPipelineNotification::notifyProcessed,
571 package->info.name(),
572 package->info.relativePath(),
573 QString(),
574 package->faceRects.size(),
575 package->thumbnail);
576
577 // delete the package
578
579 delete package;
580
581 package = nullptr;
582
583 if (
584 (0 == queues[MLPipelineStage::Loader]->size()) &&
585 (0 == queues[MLPipelineStage::Extractor]->size()) &&
586 (0 == queues[MLPipelineStage::Writer]->size())
587 )
588 {
589 Q_EMIT progressValueChanged((float)1.0);
590
591 totalItemCount = 0;
592
593 Q_EMIT finished();
594 }
595 else
596 {
597 ++itemsProcessed;
598
599 Q_EMIT progressValueChanged((float)itemsProcessed/(float)totalItemCount);
600 }
601 }
602
603 /* =========================================================================================
604 * End pipeline stage specific loop
605 */
606
607 MLPIPELINE_LOOP_END(MLPipelineStage::Writer, "FacePipelineEdit::writer");
608
609 /* =========================================================================================
610 * Pipeline stage specific cleanup
611 *
612 * Use the block from here to MLPIPELINE_STAGE_END to clean up any resources used by the stage.
613 */
614
615 MLPIPELINE_STAGE_END(MLPipelineStage::Writer, MLPipelineStage::None);
616 }
617
618 void FacePipelineEdit::addMoreWorkers()
619 {
620 // edit pipeline is always single thread per stage. Do nothing.
621 }
622
623 FaceTagsIface FacePipelineEdit::getRejectedFaceTagList(const FaceTagsIface& face) const
624 {
625 /**
626 * some FaceTagsIface objects don't have the rejectedFaceTagList set
627 * because of the way they were loaded.
628 * This function will set it to the value from the database.
629 */
630
631 FaceTagsIface newFace(face);
632
633 QList<FaceTagsIface> faces = FaceUtils().databaseFaces(face.imageId());
634
635 for (const FaceTagsIface& f : std::as_const(faces))
636 {
637 if (f.region() == newFace.region())
638 {
639 newFace.setRejectedFaceTagList(f.rejectedFaceTagList());
640 break;
641 }
642 }
643
644 return newFace;
645 }
646
647 } // namespace Digikam
648
649 #include "moc_facepipelineedit.cpp"