Debian Codegraph @

core/utilities/facemanagement/pipelines/facepipelinebase.cpp

raw
1 /* ============================================================
2 *
3 * This file is a part of digiKam project
4 * https://www.digikam.org
5 *
6 * Date : 2024-11-10
7 * Description : Integrated, multithread face detection / recognition
8 *
9 * SPDX-FileCopyrightText: 2024-2026 by Gilles Caulier <caulier dot gilles at gmail dot com>
10 * SPDX-FileCopyrightText: 2024-2025 by Michael Miller <michael underscore miller at msn dot com>
11 *
12 * SPDX-License-Identifier: GPL-2.0-or-later
13 *
14 * ============================================================ */
15
16 #include "facepipelinebase.h"
17
18 // Qt includes
19
20 #include <QList>
21
22 // KDE includes
23
24 #include <klocalizedstring.h>
25
26 // Local includes
27
28 #include "digikam_debug.h"
29 #include "digikam_opencv.h"
30 #include "sharedqueue.h"
31 #include "facepipelinepackagebase.h"
32 #include "thumbnailloadthread.h"
33 #include "dnnsfaceextractor.h"
34 #include "dnnmodelmanager.h"
35 #include "identityprovider.h"
36
37 namespace Digikam
38 {
39
40 FacePipelineBase::FacePipelineBase(const FaceScanSettings& _settings)
41 : MLPipelineFoundation(),
42 settings (_settings)
43 {
44 }
45
46 FacePipelineBase::~FacePipelineBase()
47 {
48 }
49
50 double FacePipelineBase::detectNoise1(const cv::Mat& cvGrayImage) const
51 {
52 // Use a Gaussian filter to detect noise
53
54 // apply Gaussian blur to the image
55
56 cv::Mat blurred;
57 cv::GaussianBlur(cvGrayImage, blurred, cv::Size(5, 5), 0);
58
59 // calculate the difference between the original and blurred image
60
61 cv::Mat noise = cvGrayImage - blurred;
62
63 // calculate the standard deviation of the noise
64
65 cv::Mat mean, stddev;
66 cv::meanStdDev(noise, mean, stddev);
67
68 double noiseLevel = stddev.at<double>(0, 0);
69
70 return noiseLevel;
71 }
72
73 double FacePipelineBase::detectNoise2(const cv::Mat& cvGrayImage) const
74 {
75 int H = cvGrayImage.rows;
76 int W = cvGrayImage.cols;
77 cv::Mat M = (cv::Mat_<double>(3, 3) << 1, -2, 1,
78 -2, 4, -2,
79 1, -2, 1);
80
81 cv::Mat convolved;
82 cv::filter2D(cvGrayImage, convolved, CV_64F, M);
83
84 double sigma = cv::sum(cv::abs(convolved))[0];
85 sigma = sigma * std::sqrt(0.5 * M_PI) / (6 * (W - 2) * (H - 2));
86
87 return sigma;
88 }
89
90 double FacePipelineBase::detectBlur(const cv::Mat& cvGrayImage) const
91 {
92 // Use a Fast Fourier Transform to detect blurriness
93
94 int cx = cvGrayImage.cols / 2;
95 int cy = cvGrayImage.rows / 2;
96
97 // Convert the image to a flat float
98
99 cv::Mat fImage;
100 cvGrayImage.convertTo(fImage, CV_32F);
101
102 // FFT
103
104 cv::Mat fourierTransform;
105 cv::dft(fImage, fourierTransform, cv::DFT_SCALE | cv::DFT_COMPLEX_OUTPUT);
106
107 // center low frequencies in the middle
108 // by shuffling the quadrants.
109
110 cv::Mat q0(fourierTransform, cv::Rect(0, 0, cx, cy)); // Top-Left - Create a ROI per quadrant
111 cv::Mat q1(fourierTransform, cv::Rect(cx, 0, cx, cy)); // Top-Right
112 cv::Mat q2(fourierTransform, cv::Rect(0, cy, cx, cy)); // Bottom-Left
113 cv::Mat q3(fourierTransform, cv::Rect(cx, cy, cx, cy)); // Bottom-Right
114
115 // swap quadrants (Top-Left with Bottom-Right)
116
117 cv::Mat tmp;
118 q0.copyTo(tmp);
119 q3.copyTo(q0);
120 tmp.copyTo(q3);
121
122 // swap quadrant (Top-Right with Bottom-Left)
123
124 q1.copyTo(tmp);
125 q2.copyTo(q1);
126 tmp.copyTo(q2);
127
128 // Block the low frequencies
129 // #define BLOCK could also be a argument on the command line of course
130
131 fourierTransform(cv::Rect(cx - BLOCK, cy - BLOCK, 2 * BLOCK, 2 * BLOCK)).setTo(0);
132
133 //shuffle the quadrants to their original position
134
135 cv::Mat orgFFT;
136 fourierTransform.copyTo(orgFFT);
137 cv::Mat p0(orgFFT, cv::Rect(0, 0, cx, cy)); // Top-Left - Create a ROI per quadrant
138 cv::Mat p1(orgFFT, cv::Rect(cx, 0, cx, cy)); // Top-Right
139 cv::Mat p2(orgFFT, cv::Rect(0, cy, cx, cy)); // Bottom-Left
140 cv::Mat p3(orgFFT, cv::Rect(cx, cy, cx, cy)); // Bottom-Right
141
142 // swap quadrant (Top-Left with Bottom-Right)
143
144 p0.copyTo(tmp);
145 p3.copyTo(p0);
146 tmp.copyTo(p3);
147
148 // swap quadrant (Top-Right with Bottom-Left)
149
150 p1.copyTo(tmp);
151 p2.copyTo(p1);
152 tmp.copyTo(p2);
153
154 // IFFT
155
156 cv::Mat invFFT;
157 cv::Mat logFFT;
158 double minVal = 0.0;
159 double maxVal = 0.0;
160
161 cv::dft(orgFFT, invFFT, cv::DFT_INVERSE | cv::DFT_REAL_OUTPUT);
162
163 invFFT = cv::abs(invFFT);
164 cv::minMaxLoc(invFFT, &minVal, &maxVal);
165
166 // check for impossible values
167
168 if (maxVal <= 0.0)
169 {
170 return 1;
171 }
172
173 cv::log(invFFT, logFFT);
174 logFFT *= 20;
175
176 cv::Scalar result = cv::mean(logFFT);
177
178 return result.val[0];
179 }
180
181 bool FacePipelineBase::useForTraining(const QSize& faceSize, const cv::Mat& cvImage)
182 {
183 if (!detectorModel)
184 {
185 detectorModel = DNNModelManager::instance()->getModel(QStringLiteral("yunet"),
186 DNNModelUsage::DNNUsageFaceDetection);
187 }
188
189 // thumbnail must be at least minThumbnailSize of the size the detector expects
190
191 if (
192 ((detectorModel->info.imageSize * minThumbnailSize) > faceSize.width()) ||
193 ((detectorModel->info.imageSize * minThumbnailSize) > faceSize.height())
194 )
195 {
196 qCDebug(DIGIKAM_FACESENGINE_LOG) << "FacePipelineBase::useForTraining Image is too small for face detection"
197 << faceSize.width() << "x" << faceSize.height()
198 << "required:" << detectorModel->info.imageSize * minThumbnailSize;
199 return false;
200 }
201
202 // convert to grayscale for use in noise and blur detection
203
204 cv::Mat cvGrayImage;
205 cv::cvtColor(cvImage, cvGrayImage, cv::COLOR_RGB2GRAY);
206
207 // use a Gaussian filter to check for noisy images
208
209 double noise1 = detectNoise1(cvGrayImage);
210 double noise2 = detectNoise2(cvGrayImage);
211
212 // check if image is too noisy
213
214 if ((noise1 > noiseThreshold1) || (noise2 > noiseThreshold2))
215 {
216 return false;
217 }
218
219 // use a FFT filter to check for blurred images
220
221 double blur = detectBlur(cvGrayImage);
222
223 // check if image is too blurry
224
225 if (blur < blurThreshold)
226 {
227 return false;
228 }
229
230 return true;
231 }
232
233 bool FacePipelineBase::commonFaceThumbnailLoader(const QString& pipelineName,
234 QThread::Priority stagePriority,
235 MLPipelineStage thisStage,
236 MLPipelineStage nextStage)
237 {
238 MLPIPELINE_STAGE_START(stagePriority, thisStage, nextStage);
239 FacePipelinePackageBase* package = nullptr;
240
241 /* =========================================================================================
242 * Pipeline stage specific initialization code
243 *
244 * Use the block from here to MLPIPELINE_LOOP_START to initialize the stage.
245 * The code in this block is run once per stage initialization. The number of instances
246 * is at least 1. More instances are created by addMoreWorkers if needed.
247 */
248
249 ThumbnailLoadThread* const thumbnailLoadThread = new ThumbnailLoadThread;
250 /*
251 ThumbnailLoadThread* thumbnailLoadThread = ThumbnailLoadThread::defaultThread();
252 */
253 thumbnailLoadThread->setPixmapRequested(false);
254 thumbnailLoadThread->setThumbnailSize(ThumbnailLoadThread::maximumThumbnailSize());
255 thumbnailLoadThread->setPriority(QThread::NormalPriority);
256
257 ThumbnailImageCatcher* const catcher = new ThumbnailImageCatcher(thumbnailLoadThread);
258
259 catcher->setActive(true);
260
261 MLPIPELINE_LOOP_START(thisStage, thisQueue);
262 package = static_cast<FacePipelinePackageBase*>(mlpackage);
263
264 /* =========================================================================================
265 * Start pipeline stage specific loop
266 *
267 * All code from here to MLPIPELINE_LOOP_END is in a try/catch block and loop.
268 * This loop is run once per image.
269 */
270
271 {
272 catcher->thread()->find(ItemInfo::thumbnailIdentifier(package->face.imageId()),
273 package->face.region().toRect());
274 catcher->enqueue();
275 QList<QImage> images = catcher->waitForThumbnails();
276
277 if (images.size() && !images[0].isNull())
278 {
279 // store the thumbnail image in the package
280
281 package->thumbnail = images[0];
282
283 // create a thumbnail icon
284
285 package->thumbnailIcon = QIcon(DImg(package->thumbnail).smoothScale(48,
286 48,
287 Qt::KeepAspectRatio).convertToPixmap());
288
289 // send the package to the next stage
290
291 enqueue(nextQueue, package);
292
293 package = nullptr;
294 }
295 else
296 {
297 // send a notification that the file was skipped
298
299 notify(MLPipelineNotification::notifySkipped,
300 package->info.name(),
301 package->info.filePath(),
302 QString(),
303 0,
304 QIcon::fromTheme(QLatin1String("image-missing")));
305
306 // delete the package since it is not needed
307
308 delete package;
309
310 package = nullptr;
311 }
312 }
313
314 /* =========================================================================================
315 * End pipeline stage specific loop
316 */
317
318 MLPIPELINE_LOOP_END(thisStage, QString(pipelineName + QLatin1String("::commonFaceThumbnailLoader")).toLocal8Bit().data());
319
320 /* =========================================================================================
321 * Pipeline stage specific cleanup
322 *
323 * Use the block from here to MLPIPELINE_STAGE_END to clean up any resources used by the stage.
324 */
325
326 catcher->setActive(false);
327 catcher->thread()->stopAllTasks();
328 catcher->cancel();
329
330 delete catcher->thread();
331 delete catcher;
332
333 MLPIPELINE_STAGE_END(thisStage, nextStage);
334 }
335
336 bool FacePipelineBase::commonFaceThumbnailExtractor(const QString& pipelineName,
337 QThread::Priority stagePriority,
338 MLPipelineStage thisStage,
339 MLPipelineStage nextStage,
340 bool trainingQualityCheck)
341 {
342 MLPIPELINE_STAGE_START(stagePriority, thisStage, nextStage);
343 FacePipelinePackageBase* package = nullptr;
344
345 /* =========================================================================================
346 * Pipeline stage specific initialization code
347 *
348 * Use the block from here to MLPIPELINE_LOOP_START to initialize the stage.
349 * The code in this block is run once per stage initialization. The number of instances
350 * is at least 1. More instances are created by addMoreWorkers if needed.
351 */
352
353 DNNSFaceExtractor extractor;
354 FaceUtils utils;
355
356 MLPIPELINE_LOOP_START(thisStage, thisQueue);
357 package = static_cast<FacePipelinePackageBase*>(mlpackage);
358
359 /* =========================================================================================
360 * Start pipeline stage specific loop
361 *
362 * All code from here to MLPIPELINE_LOOP_END is in a try/catch block and loop.
363 * This loop is run once per image.
364 */
365
366 {
367 QImage inputImage(package->thumbnail.copy());
368
369 // preprocess image to be in the correct format
370
371 if (inputImage.format() != QImage::Format_BGR888)
372 {
373 inputImage = inputImage.convertToFormat(QImage::Format_BGR888);
374 }
375
376 // create a cv::Mat image from the QImage
377
378 cv::Mat cvImage = cv::Mat(
379 inputImage.height(),
380 inputImage.width(),
381 CV_8UC3,
382 inputImage.scanLine(0),
383 inputImage.bytesPerLine()
384 );
385
386 if (!cvImage.empty())
387 {
388 cv::UMat cvUImage = cvImage.getUMat(cv::ACCESS_READ);
389
390 // Extract the face features
391
392 QPair<cv::Mat, cv::Mat> faceEmbedding = extractor.getFaceEmbedding(cvUImage);
393 package->features = faceEmbedding.second;
394
395 // Check for a valid feature set
396
397 if (!package->features.empty() && trainingQualityCheck)
398 {
399 /**
400 * Compute the size of the face region in pixels as shown in the original image
401 * This is used to check if the image is suitable for training.
402 */
403
404 // Prevent divide by zero - bug 513760.
405
406 float width = static_cast<float>(package->info.dimensions().width());
407 float height = static_cast<float>(package->info.dimensions().height());
408 float thumbWidth = static_cast<float>(package->thumbnail.width());
409 float thumbHeight = static_cast<float>(package->thumbnail.height());
410
411 if ((width <= 0.0) || (height <= 0.0) || (thumbWidth <= 0.0) || (thumbHeight <= 0.0))
412 {
413 qCWarning(DIGIKAM_FACESENGINE_LOG) << "Invalid dimensions or thumbnail size, skipping training check.";
414 package->useForTraining = false;
415 }
416 else
417 {
418 // Percentage of the region compared to the image
419
420 float xThumbnailPercent = static_cast<float>(package->face.region().toRect().width()) / width;
421 float yThumbnailPercent = static_cast<float>(package->face.region().toRect().height()) / height;
422
423 // Prevent infinite or not a number values - bug 513760.
424
425 float xFacePos = faceEmbedding.first.at<float>(0, 2);
426 float yFacePos = faceEmbedding.first.at<float>(0, 3);
427
428 if (
429 std::isnan(xFacePos) || std::isnan(yFacePos) ||
430 std::isinf(xFacePos) || std::isinf(yFacePos)
431 )
432 {
433 qCWarning(DIGIKAM_FACESENGINE_LOG) << "Invalid face embedding values, skipping training check.";
434 package->useForTraining = false;
435 }
436 else
437 {
438 // Percentage of the face compared to the thumbnail
439
440 float xFaceAreaPercent = xFacePos / thumbWidth;
441 float yFaceAreaPercent = yFacePos / thumbHeight;
442
443 // Compute the size of the face area in pixels
444
445 QSize faceSize(qRound(package->info.dimensions().width() * xThumbnailPercent * xFaceAreaPercent),
446 qRound(package->info.dimensions().height() * yThumbnailPercent * yFaceAreaPercent));
447
448 // Check if the image is suitable for training
449
450 package->useForTraining = useForTraining(faceSize, cvImage);
451 }
452 }
453 }
454
455 for (const auto tagId : package->face.rejectedFaceTagList())
456 {
457 // add the Identity ID for the rejected face tag to the exclusion list
458
459 QMultiMap<QString, QString> attributes = FaceTags::identityAttributes(tagId);
460 Identity identity = IdentityProvider::instance()->findIdentity(attributes);
461
462 if (!identity.isNull())
463 {
464 package->exclusionIdentityIds << identity.id();
465 }
466 }
467
468 // send the package to the next stage
469
470 enqueue(nextQueue, package);
471
472 package = nullptr;
473 }
474 else
475 {
476 // send a notification that the file was skipped
477
478 notify(MLPipelineNotification::notifySkipped,
479 package->info.name(),
480 package->info.filePath(),
481 QString(),
482 0,
483 package->thumbnailIcon);
484
485 // delete the package since it is not needed
486
487 delete package;
488
489 package = nullptr;
490 }
491 }
492
493 /* =========================================================================================
494 * End pipeline stage specific loop
495 */
496
497 MLPIPELINE_LOOP_END(thisStage, QString(pipelineName + QLatin1String("::commonFaceThumbnailExtractor")).toLocal8Bit().data());
498
499 /* =========================================================================================
500 * Pipeline stage specific cleanup
501 *
502 * Use the block from here to MLPIPELINE_STAGE_END to clean up any resources used by the stage.
503 */
504
505 MLPIPELINE_STAGE_END(thisStage, nextStage);
506 }
507
508 bool FacePipelineBase::enqueue(MLPipelineQueue* thisQueue, MLPipelinePackageFoundation* package)
509 {
510 if (nullptr != package)
511 {
512 // calculate the package size. Only big items need to be checked
513
514 package->size = static_cast<FacePipelinePackageBase*>(package)->image.size().width() *
515 static_cast<FacePipelinePackageBase*>(package)->image.size().height() * 4;
516 }
517
518 return MLPipelineFoundation::enqueue(thisQueue, package);
519 }
520
521 } // namespace Digikam
522
523 #include "moc_facepipelinebase.cpp"