Debian Codegraph @

core/utilities/facemanagement/pipelines/recognize/facepipelinerecognize.cpp

raw
1 /* ============================================================
2 *
3 * This file is a part of digiKam project
4 * https://www.digikam.org
5 *
6 * Date : 2024-11-10
7 * Description : Performs face detection and recognition
8 *
9 * SPDX-FileCopyrightText: 2024-2026 by Gilles Caulier <caulier dot gilles at gmail dot com>
10 * SPDX-FileCopyrightText: 2024-2025 by Michael Miller <michael underscore miller at msn dot com>
11 *
12 * SPDX-License-Identifier: GPL-2.0-or-later
13 *
14 * ============================================================ */
15
16 #include "facepipelinerecognize.h"
17
18 // Qt includes
19
20 #include <QList>
21 #include <QSet>
22 #include <QElapsedTimer>
23 #include <QRectF>
24
25 // KDE includes
26
27 #include <klocalizedstring.h>
28
29 // Local includes
30
31 #include "digikam_debug.h"
32 #include "digikam_opencv.h"
33 #include "sharedqueue.h"
34 #include "collectionmanager.h"
35 #include "album.h"
36 #include "iteminfo.h"
37 #include "coredb.h"
38 #include "facescansettings.h"
39 #include "dimg.h"
40 #include "faceutils.h"
41 #include "facepipelinepackagebase.h"
42 #include "identityprovider.h"
43 #include "identity.h"
44 #include "dnnsfaceextractor.h"
45 #include "faceclassifier.h"
46 #include "metadatahub.h"
47 #include "scancontroller.h"
48
49 namespace Digikam
50 {
51
52 FacePipelineRecognize::FacePipelineRecognize(const FaceScanSettings& _settings)
53 : FacePipelineBase(_settings)
54 {
55 }
56
57 bool FacePipelineRecognize::start()
58 {
59 FacePipelineBase::start();
60
61 QVariantMap params;
62 params[QLatin1String("detectAccuracy")] = settings.detectAccuracy;
63 params[QLatin1String("detectModel")] = settings.detectModel;
64 params[QLatin1String("detectSize")] = settings.detectSize;
65 params[QLatin1String("recognizeAccuracy")] = settings.recognizeAccuracy;
66 params[QLatin1String("recognizeModel")] = settings.recognizeModel;
67 detector.setParameters(params);
68
69 {
70 // use the mutex to synchronize the start of the threads
71
72 QMutexLocker lock(&mutex);
73
74 // add the worker threads for this pipeline
75
76 addWorker(MLPipelineStage::Finder);
77 addWorker(MLPipelineStage::Loader);
78 addWorker(MLPipelineStage::Extractor);
79 addWorker(MLPipelineStage::Classifier);
80 addWorker(MLPipelineStage::Writer);
81 }
82
83 return true;
84 }
85
86 bool FacePipelineRecognize::finder()
87 {
88 ActionThreadBase::setCurrentThreadName(QString::fromLatin1("FaceReco%1").arg(QLatin1String(__FUNCTION__))); // To customize thread name
89
90 MLPIPELINE_FINDER_START(settings.workerThreadPriority, MLPipelineStage::Loader);
91
92 /* =========================================================================================
93 * Pipeline finder specific initialization code
94 *
95 * Use the block from here to MLPIPELINE_FINDER_END to find the IDs images to process.
96 * The code in this block is run once per stage initialization. The number of instances
97 * is alaways 1.
98 */
99
100 // get the IDs to process
101
102 FaceUtils utils;
103 QSet<qlonglong> filter;
104 int serialNumber = 0;
105
106 for (const Album* const album : std::as_const(settings.albums))
107 {
108 if (cancelled)
109 {
110 break;
111 }
112
113 if (!album->isTrashAlbum())
114 {
115 QList<qlonglong> imageIds;
116
117 // get the image IDs for the album
118
119 if (album->type() == Album::PHYSICAL)
120 {
121 imageIds = CoreDbAccess().db()->getImageIds(album->id(),
122 DatabaseItem::Status::Visible,
123 true);
124 }
125 else if (album->type() == Album::TAG)
126 {
127 imageIds = CoreDbAccess().db()->getItemIDsInTag(album->id());
128 }
129
130 // iterate over the image IDs and add unique IDs to the queue for processing
131
132 for (qlonglong imageId : std::as_const(imageIds))
133 {
134 // filter out duplicate image IDs
135
136 if (!filter.contains(imageId))
137 {
138 // quick check if we should add threads.
139
140 if (!moreCpu)
141 {
142 moreCpu = checkMoreWorkers(totalItemCount, imageIds.size(), settings.useFullCpu);
143 }
144
145 QList<FaceTagsIface> faces = utils.unconfirmedFaceTagsIfaces(imageId);
146
147 for (const FaceTagsIface& face : std::as_const(faces))
148 {
149 ++totalItemCount;
150 filter << imageId;
151 enqueue(nextQueue, new FacePipelinePackageBase(imageId, face, ++serialNumber));
152 }
153 }
154 }
155 }
156 }
157
158 for (const ItemInfo& info : std::as_const(settings.infos))
159 {
160 // filter out duplicate image IDs
161
162 qlonglong imageId = info.id();
163
164 if (!filter.contains(imageId))
165 {
166 QList<FaceTagsIface> faces = utils.unconfirmedFaceTagsIfaces(imageId);
167
168 for (const FaceTagsIface& face : std::as_const(faces))
169 {
170 ++totalItemCount;
171 filter << imageId;
172 enqueue(nextQueue, new FacePipelinePackageBase(imageId, face, ++serialNumber));
173 }
174 }
175 }
176
177 /* =========================================================================================
178 * Pipeline finder specific cleanup
179 *
180 * Use the block from here to MLPIPELINE_FINDER_END to clean up any resources used by the stage.
181 */
182
183 MLPIPELINE_FINDER_END(MLPipelineStage::Loader);
184 }
185
186 bool FacePipelineRecognize::loader()
187 {
188 ActionThreadBase::setCurrentThreadName(QString::fromLatin1("FaceReco%1").arg(QLatin1String(__FUNCTION__))); // To customize thread name
189
190 return commonFaceThumbnailLoader(QStringLiteral("FacePipelineRecognize"),
191 settings.workerThreadPriority,
192 MLPipelineStage::Loader,
193 MLPipelineStage::Extractor);
194 }
195
196 bool FacePipelineRecognize::extractor()
197 {
198 ActionThreadBase::setCurrentThreadName(QString::fromLatin1("FaceReco%1").arg(QLatin1String(__FUNCTION__))); // To customize thread name
199
200 return commonFaceThumbnailExtractor(QStringLiteral("FacePipelineRecognize"),
201 settings.workerThreadPriority,
202 MLPipelineStage::Extractor,
203 MLPipelineStage::Classifier);
204 }
205
206 bool FacePipelineRecognize::classifier()
207 {
208 ActionThreadBase::setCurrentThreadName(QString::fromLatin1("FaceReco%1").arg(QLatin1String(__FUNCTION__))); // To customize thread name
209
210 MLPIPELINE_STAGE_START(settings.workerThreadPriority, MLPipelineStage::Classifier, MLPipelineStage::Writer);
211 FacePipelinePackageBase* package = nullptr;
212
213 /* =========================================================================================
214 * Pipeline stage specific initialization code
215 *
216 * Use the block from here to MLPIPELINE_LOOP_START to initialize the stage.
217 * The code in this block is run once per stage initialization. The number of instances
218 * is at least 1. More instances are created by addMoreWorkers if needed.
219 */
220
221 FaceClassifier* const classifier = FaceClassifier::instance();
222 classifier->setParameters(settings);
223
224 MLPIPELINE_LOOP_START(MLPipelineStage::Classifier, thisQueue);
225 package = static_cast<FacePipelinePackageBase*>(mlpackage);
226
227 /* =========================================================================================
228 * Start pipeline stage specific loop
229 *
230 * All code from here to MLPIPELINE_LOOP_END is in a try/catch block and loop.
231 * This loop is run once per image.
232 */
233 {
234 // verify the feature mat is the correct size
235
236 if (
237 (2 == package->features.dims) &&
238 (1 == package->features.rows) &&
239 (128 == package->features.cols)
240 )
241 {
242 // classify the features
243
244 package->label = classifier->predict(package->features, package->exclusionIdentityIds);
245 }
246
247 enqueue(nextQueue, package);
248
249 package = nullptr;
250 }
251 /* =========================================================================================
252 * End pipeline stage specific loop
253 */
254
255 MLPIPELINE_LOOP_END(MLPipelineStage::Classifier, "FacePipelineRecognize::classifier");
256
257 /* =========================================================================================
258 * Pipeline stage specific cleanup
259 *
260 * Use the block from here to MLPIPELINE_STAGE_END to clean up any resources used by the stage.
261 */
262
263 MLPIPELINE_STAGE_END(MLPipelineStage::Classifier, MLPipelineStage::Writer);
264 }
265
266 bool FacePipelineRecognize::writer()
267 {
268 ActionThreadBase::setCurrentThreadName(QString::fromLatin1("FaceReco%1").arg(QLatin1String(__FUNCTION__))); // To customize thread name
269
270 MLPIPELINE_STAGE_START(settings.workerThreadPriority, MLPipelineStage::Writer, MLPipelineStage::None);
271 FacePipelinePackageBase* package = nullptr;
272
273 /* =========================================================================================
274 * Pipeline stage specific initialization code
275 *
276 * Use the block from here to MLPIPELINE_LOOP_START to initialize the stage.
277 * The code in this block is run once per stage initialization. The number of instances
278 * is at least 1. More instances are created by addMoreWorkers if needed.
279 */
280
281 FaceUtils utils;
282 const IdentityProvider* const idProvider = IdentityProvider::instance();
283
284 MLPIPELINE_LOOP_START(MLPipelineStage::Writer, thisQueue);
285 package = static_cast<FacePipelinePackageBase*>(mlpackage);
286
287 /* =========================================================================================
288 * Start pipeline stage specific loop
289 *
290 * All code from here to MLPIPELINE_LOOP_END is in a try/catch block and loop.
291 * This loop is run once per image.
292 */
293 {
294 QString displayName;
295 int matches = 0;
296
297 if (FaceClassifier::UNKNOWN_LABEL_ID != package->label)
298 {
299 Identity identity = idProvider->identity(package->label);
300 int tagId = FaceTags::getOrCreateTagForIdentity(identity.attributesMap());
301
302 if (package->face.tagId() != tagId)
303 {
304 utils.changeSuggestedName(package->face, tagId);
305 }
306
307 displayName += identity.attribute(QStringLiteral("name"));
308 matches = 1;
309 }
310 else
311 {
312 if (package->face.isUnconfirmedName())
313 {
314 utils.changeTag(package->face, FaceTags::unknownPersonTagId());
315 }
316 }
317
318 // if (utils.normalTagChanged())
319 // {
320 // MetadataHub hub;
321 // hub.load(package->info);
322
323 // ScanController::FileMetadataWrite writeScope(package->info);
324 // writeScope.changed(hub.writeToMetadata(package->info, MetadataHub::WRITE_TAGS));
325 // }
326
327 QString albumName = CollectionManager::instance()->albumRootLabel(package->info.albumRootId());
328
329 // send a notification that the image was processed
330
331 notify(MLPipelineNotification::notifyProcessed,
332 package->info.name(),
333 albumName + package->info.relativePath(),
334 displayName,
335 matches,
336 package->thumbnail);
337
338 // delete the package
339
340 delete package;
341
342 package = nullptr;
343 }
344
345 /* =========================================================================================
346 * End pipeline stage specific loop
347 */
348
349 MLPIPELINE_LOOP_END(MLPipelineStage::Writer, "FacePipelineRecognize::writer");
350
351 /* =========================================================================================
352 * Pipeline stage specific cleanup
353 *
354 * Use the block from here to MLPIPELINE_STAGE_END to clean up any resources used by the stage.
355 */
356
357 MLPIPELINE_STAGE_END(MLPipelineStage::Writer, MLPipelineStage::None);
358 }
359
360 void FacePipelineRecognize::addMoreWorkers()
361 {
362 // use the performanceProfile metrics to find the slowest stages
363 // and add more workers to those stages
364
365 // for the recognition pipeline, the extractor is the slowest stage
366 // so add 1 more loader and 2 more extractors
367
368 addWorker(Loader);
369 addWorker(Extractor);
370 addWorker(Extractor);
371 }
372
373 } // namespace Digikam
374
375 #include "moc_facepipelinerecognize.cpp"