Avant d’utiliser Digikam, il est plus facile de supprimer les images identiques via un simple script :
find Images/ -type f -exec md5sum '{}' ';' | sort | uniq --all-repeated=separate -w 15 > dupes.txt
awk '/^$/{getline;print;}' dupes.txt | awk '{print $2 " " $3 " " $4}' | xargs gvfs-trash {}
Il est aussi possible d’utiliser :
fdupes -rSm Images
L’option -d permet la suppression.
Ensuite on peut utiliser Digikam, qui lui permet de reconnaitre des images identiques mais n’ayant pas la même taille.
J’ai fait une petite lecture des tables de Dikikam afin de faire un export des images similaires avec un taux à 1.0 :
$ sqlite3
SQLite version 3.22.0 2018-01-22 18:45:57
Enter ".help" for usage hints.
Connected to a transient in-memory database.
Use ".open FILENAME" to reopen on a persistent database.
sqlite> .open similarity.db
sqlite> .tables
ImageHaarMatrix ImageSimilarity SimilaritySettings
sqlite> .schema ImageSimilarity
CREATE TABLE ImageSimilarity
(imageid1 INTEGER NOT NULL,
imageid2 INTEGER NOT NULL,
algorithm INTEGER,
value DOUBLE,
CONSTRAINT Similar UNIQUE(imageid1, imageid2, algorithm));
sqlite> .schema ImageHaarMatrix
CREATE TABLE ImageHaarMatrix
(imageid INTEGER PRIMARY KEY,
modificationDate DATETIME,
uniqueHash TEXT,
matrix BLOB);
CREATE TRIGGER delete_similarities DELETE ON ImageHaarMatrix
BEGIN
DELETE FROM ImageSimilarity
WHERE ( ImageSimilarity.imageid1=OLD.imageid OR ImageSimilarity.imageid2=OLD.imageid )
AND ( ImageSimilarity.algorithm=1 );
END;
sqlite> .schema SimilaritySettings
CREATE TABLE SimilaritySettings
(keyword TEXT NOT NULL UNIQUE,
value TEXT);
sqlite> .open digikam4.db
sqlite> .tables
AlbumRoots ImageHistory ImageRelations Settings
Albums ImageInformation ImageTagProperties TagProperties
DownloadHistory ImageMetadata ImageTags Tags
ImageComments ImagePositions Images TagsTree
ImageCopyright ImageProperties Searches VideoMetadata
sqlite> .schema Images
CREATE TABLE Images
(id INTEGER PRIMARY KEY,
album INTEGER,
name TEXT NOT NULL,
status INTEGER NOT NULL,
category INTEGER NOT NULL,
modificationDate DATETIME,
fileSize INTEGER,
uniqueHash TEXT,
manualOrder INTEGER,
UNIQUE (album, name));
CREATE INDEX dir_index ON Images (album);
CREATE INDEX hash_index ON Images (uniqueHash);
CREATE INDEX image_name_index ON Images (name);
CREATE TRIGGER delete_image DELETE ON Images
BEGIN
DELETE FROM ImageTags WHERE imageid=OLD.id;
DELETE From ImageInformation WHERE imageid=OLD.id;
DELETE From ImageMetadata WHERE imageid=OLD.id;
DELETE From VideoMetadata WHERE imageid=OLD.id;
DELETE From ImagePositions WHERE imageid=OLD.id;
DELETE From ImageComments WHERE imageid=OLD.id;
DELETE From ImageCopyright WHERE imageid=OLD.id;
DELETE From ImageProperties WHERE imageid=OLD.id;
DELETE From ImageHistory WHERE imageid=OLD.id;
DELETE FROM ImageRelations WHERE subject=OLD.id OR object=OLD.id;
DELETE FROM ImageTagProperties WHERE imageid=OLD.id;
UPDATE Albums SET icon=null WHERE icon=OLD.id;
UPDATE Tags SET icon=null WHERE icon=OLD.id;
END;
sqlite> .schema ImageInformation
CREATE TABLE ImageInformation
(imageid INTEGER PRIMARY KEY,
rating INTEGER,
creationDate DATETIME,
digitizationDate DATETIME,
orientation INTEGER,
width INTEGER,
height INTEGER,
format TEXT,
colorDepth INTEGER,
colorModel INTEGER);
CREATE INDEX creationdate_index ON ImageInformation (creationDate);
sqlite> .schema Albums
CREATE TABLE Albums
(id INTEGER PRIMARY KEY,
albumRoot INTEGER NOT NULL,
relativePath TEXT NOT NULL,
date DATE,
caption TEXT,
collection TEXT,
icon INTEGER,
UNIQUE(albumRoot, relativePath));
CREATE TRIGGER delete_album DELETE ON Albums
BEGIN
DELETE FROM Images
WHERE Images.album = OLD.id;
END;
sqlite> attach 'digikam4.db' as db1;
sqlite> attach 'similarity.db' as db2;
sqlite> select count(*) from db1.Images as A, db2.ImageSimilarity as B, db1.Albums as C where B.imageid2 = A.id and B.algorithm = 1.0 and A.album = C.id;
36796
sqlite> select relativePath || '/' || name from db1.Images as A, db2.ImageSimilarity as B, db1.Albums as C where B.imageid2 = A.id and B.algorithm = 1.0 and A.album = C.id group by relativePath;
...
sqlite> .output file_duplicate.txt
sqlite> select '.' || relativePath || '/' || name from db1.Images as A, db2.ImageSimilarity as B, db1.Albums as C where B.imageid2 = A.id and B.algorithm = 1.0 and A.album = C.id group by relativePath;
sqlite> select count(*) from db1.Images as A, db2.ImageSimilarity as B, db1.Albums as C where B.imageid2 = A.id and A.album = C.id and relativePath = '/2019/11/28';
654
sqlite> select count(*) from db1.Images as A, db2.ImageSimilarity as B, db1.Albums as C where B.imageid1 = A.id and A.album = C.id and relativePath = '/2019/11/28';
2545
sqlite> .output file_duplicate_2.txt
sqlite> select '.' || relativePath || '/' || name from db1.Images as A, db2.ImageSimilarity as B, db1.Albums as C where B.imageid2 = A.id and B.algorithm > 0.96 and A.album = C.id;
sqlite> .output file_duplicate_3.txt
sqlite> select '.' || relativePath || '/' || name from db1.Images as A, db2.ImageSimilarity as B, db1.Albums as C where B.imageid1 = A.id and B.algorithm > 0.96 and A.album = C.id;
sqlite> .quit
Ensuite pour la suppression j’ai fait :
Sous Mac OS j’avais fait plusieurs articles …Mais sous Ubuntu c’est plus simple, voici les commandes que j’utilise :
$ du -sh Images/
$ find Images/ -iname '*.jpg' -exec mogrify \{} -verbose -resize 1920x1080\> \{} \;
$
Je considère que la résolution max est 1920x1080 (on peut aussi prendre 2048x1536). Il est vivement conseillé d’avoir un backup avant de lancer les commandes. Et je vous conseille cette lecture : https://fr.wikipedia.org/wiki/Impression_photo_num%C3%A9rique
A noter aussi que je déplace toutes les vidéos avant de lancer la commandes :
Suite à la compression via Digikam, toutes les nouvelles images compressées ont eu une mauvaise date. Ce qui donne sur le timeline ceci :
Le pic de 2019, n’a pas lieu d’être … j’ai donc essayer de lancer une ligne de commande pour changer la date de modification et de création des images.
$ time exiftool -v -r "-filemodifydate<datetimeoriginal" "-filecreateddate<datetimeoriginal" Images/
...
3214 directories scanned
119812 image files updated
15783 image files unchanged
203 files weren't updated due to errors
real 22m59,695s
user 22m0,499s
sys 0m21,394s
Ensuite j’ai refait un scan avec Digikam :
Je n’ai pas remarqué de changement sur la courbe … misère.
Voici l’erreur en question :
cp: cannot stat '/snap/digikam/6/kf5/usr/share/mime': No such file or directory
Directory '/root/snap/digikam/6/.local/share/mime/packages' does not exist!
ERROR: update-mime-database /root/snap/digikam/6/.local/share/mime exited abnormally with status 1
/snap/digikam/6/usr/bin/digikam: error while loading shared libraries: libQt5Sql.so.5: cannot open shared object file: No such file or directory
Voici les informations de mon système :
arias@CYBERNEURONES:~$ sudo locate libQt5Sql.so.5
/snap/kde-frameworks-5-core18/32/usr/lib/x86_64-linux-gnu/libQt5Sql.so.5
/snap/kde-frameworks-5-core18/32/usr/lib/x86_64-linux-gnu/libQt5Sql.so.5.12
/snap/kde-frameworks-5-core18/32/usr/lib/x86_64-linux-gnu/libQt5Sql.so.5.12.3
/usr/lib/x86_64-linux-gnu/libQt5Sql.so.5
/usr/lib/x86_64-linux-gnu/libQt5Sql.so.5.9
/usr/lib/x86_64-linux-gnu/libQt5Sql.so.5.9.5
arias@CYBERNEURONES:~$ ldd /snap/digikam/6/usr/bin/digikam
/snap/digikam/6/usr/bin/digikam: /usr/lib/x86_64-linux-gnu/libQt5Core.so.5: version `Qt_5.12' not found (required by /snap/digikam/6/usr/bin/digikam)
linux-vdso.so.1 (0x00007ffd447d5000)
libdigikamcore.so.6.4.0 => not found
libdigikamdatabase.so.6.4.0 => not found
libdigikamgui.so.6.4.0 => not found
libQt5Sql.so.5 => /usr/lib/x86_64-linux-gnu/libQt5Sql.so.5 (0x00007f400a869000)
libKF5I18n.so.5 => /usr/lib/x86_64-linux-gnu/libKF5I18n.so.5 (0x00007f400a61b000)
libKF5CoreAddons.so.5 => /usr/lib/x86_64-linux-gnu/libKF5CoreAddons.so.5 (0x00007f400a389000)
libKF5ConfigCore.so.5 => /usr/lib/x86_64-linux-gnu/libKF5ConfigCore.so.5 (0x00007f400a12e000)
libQt5Widgets.so.5 => /usr/lib/x86_64-linux-gnu/libQt5Widgets.so.5 (0x00007f40098e7000)
libQt5Gui.so.5 => /usr/lib/x86_64-linux-gnu/libQt5Gui.so.5 (0x00007f400917e000)
libQt5Core.so.5 => /usr/lib/x86_64-linux-gnu/libQt5Core.so.5 (0x00007f4008a33000)
libstdc++.so.6 => /usr/lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007f40086aa000)
libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007f4008492000)
libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f40080a1000)
libfam.so.0 => /usr/lib/x86_64-linux-gnu/libfam.so.0 (0x00007f4007e98000)
libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f4007c79000)
libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f40078db000)
libGL.so.1 => /usr/lib/x86_64-linux-gnu/libGL.so.1 (0x00007f400764f000)
libpng16.so.16 => /usr/lib/x86_64-linux-gnu/libpng16.so.16 (0x00007f400741d000)
libharfbuzz.so.0 => /usr/lib/x86_64-linux-gnu/libharfbuzz.so.0 (0x00007f400717f000)
libz.so.1 => /lib/x86_64-linux-gnu/libz.so.1 (0x00007f4006f62000)
libicui18n.so.60 => /usr/lib/x86_64-linux-gnu/libicui18n.so.60 (0x00007f4006ac1000)
libicuuc.so.60 => /usr/lib/x86_64-linux-gnu/libicuuc.so.60 (0x00007f400670a000)
libdouble-conversion.so.1 => /usr/lib/x86_64-linux-gnu/libdouble-conversion.so.1 (0x00007f40064f9000)
libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f40062f5000)
libglib-2.0.so.0 => /usr/lib/x86_64-linux-gnu/libglib-2.0.so.0 (0x00007f4005fde000)
/lib64/ld-linux-x86-64.so.2 (0x00007f400acc0000)
libGLX.so.0 => /usr/lib/x86_64-linux-gnu/libGLX.so.0 (0x00007f4005dad000)
libGLdispatch.so.0 => /usr/lib/x86_64-linux-gnu/libGLdispatch.so.0 (0x00007f4005af7000)
libfreetype.so.6 => /usr/lib/x86_64-linux-gnu/libfreetype.so.6 (0x00007f4005843000)
libgraphite2.so.3 => /usr/lib/x86_64-linux-gnu/libgraphite2.so.3 (0x00007f4005616000)
libicudata.so.60 => /usr/lib/x86_64-linux-gnu/libicudata.so.60 (0x00007f4003a6d000)
libpcre.so.3 => /lib/x86_64-linux-gnu/libpcre.so.3 (0x00007f40037fb000)
libX11.so.6 => /usr/lib/x86_64-linux-gnu/libX11.so.6 (0x00007f40034c3000)
libxcb.so.1 => /usr/lib/x86_64-linux-gnu/libxcb.so.1 (0x00007f400329b000)
libXau.so.6 => /usr/lib/x86_64-linux-gnu/libXau.so.6 (0x00007f4003097000)
libXdmcp.so.6 => /usr/lib/x86_64-linux-gnu/libXdmcp.so.6 (0x00007f4002e91000)
libbsd.so.0 => /lib/x86_64-linux-gnu/libbsd.so.0 (0x00007f4002c7c000)
librt.so.1 => /lib/x86_64-linux-gnu/librt.so.1 (0x00007f4002a74000)
arias@CYBERNEURONES:~$ ls -l /usr/lib/x86_64-linux-gnu/libQt5Sql.so.5
lrwxrwxrwx 1 root root 18 oct. 30 22:00 /usr/lib/x86_64-linux-gnu/libQt5Sql.so.5 -> libQt5Sql.so.5.9.5
La correction à faire :
J’ai fait un test de Digikam : https://www.digikam.org .
Quand la bibliothèque est grosse, le logiciel devient inutilisable …
Je vais attendre la version optimisé, mais c’est très prometteur.