| 1 |
#!/usr/bin/env sh |
| 2 |
|
| 3 |
# Description: List non-empty duplicates in the current dir (based on size followed by MD5) |
| 4 |
# |
| 5 |
# Source: https://www.commandlinefu.com/commands/view/3555/find-duplicate-files-based-on-size-first-then-md5-hash |
| 6 |
# |
| 7 |
# Dependencies: find md5sum sort uniq xargs gsed |
| 8 |
# |
| 9 |
# Notes: |
| 10 |
# 1. If the file size exceeds $size_digits digits the file will be misplaced |
| 11 |
# 12 digits fit files up to 931GiB |
| 12 |
# 2. Bash compatible required for mktemp |
| 13 |
# |
| 14 |
# Shell: Bash |
| 15 |
# Authors: syssyphus, KlzXS |
| 16 |
|
| 17 |
EDITOR="${EDITOR:-vi}" |
| 18 |
TMPDIR="${TMPDIR:-/tmp}" |
| 19 |
|
| 20 |
size_digits=12 |
| 21 |
tmpfile=$(mktemp "$TMPDIR/.nnnXXXXXX") |
| 22 |
|
| 23 |
printf "\ |
| 24 |
## This is an overview of all duplicate files found. |
| 25 |
## Comment out the files you wish to remove. You will be given an option to cancel. |
| 26 |
## Lines with double comments (##) are ignored. |
| 27 |
## You will have the option to remove the files with force or interactively.\n |
| 28 |
" > "$tmpfile" |
| 29 |
|
| 30 |
# shellcheck disable=SC2016 |
| 31 |
find . -size +0 -type f -printf "%${size_digits}s %p\n" | sort -rn | uniq -w"${size_digits}" -D | sed -e ' |
| 32 |
s/^ \{0,12\}\([0-9]\{0,12\}\) \(.*\)$/printf "%s %s\\n" "$(md5sum "\2")" "d\1"/ |
| 33 |
' | tr '\n' '\0' | xargs -0 -n1 sh -c | sort | { uniq -w32 --all-repeated=separate; echo; } | sed -ne ' |
| 34 |
h |
| 35 |
s/^\(.\{32\}\).* d\([0-9]*\)$/## md5sum: \1 size: \2 bytes/p |
| 36 |
g |
| 37 |
|
| 38 |
:loop |
| 39 |
N |
| 40 |
/.*\n$/!b loop |
| 41 |
p' | sed -e 's/^.\{32\} \(.*\) d[0-9]*$/\1/' >> "$tmpfile" |
| 42 |
|
| 43 |
"$EDITOR" "$tmpfile" |
| 44 |
|
| 45 |
printf "Remove commented files? (yes/no) [default=n]: " |
| 46 |
read -r commented |
| 47 |
|
| 48 |
if [ "$commented" = "y" ]; then |
| 49 |
sedcmd="/^##.*/d; /^[^#].*/d; /^$/d; s/^# *\(.*\)$/\1/" |
| 50 |
else |
| 51 |
printf "Press any key to exit" |
| 52 |
read -r _ |
| 53 |
exit |
| 54 |
fi |
| 55 |
|
| 56 |
printf "Remove with force or interactive? (f/i) [default=i]: " |
| 57 |
read -r force |
| 58 |
|
| 59 |
if [ "$force" = "f" ]; then |
| 60 |
#shellcheck disable=SC2016 |
| 61 |
sed -e "$sedcmd" "$tmpfile" | tr '\n' '\0' | xargs -0 -r sh -c 'rm -f -- "$0" "$@" </dev/tty' |
| 62 |
else |
| 63 |
#shellcheck disable=SC2016 |
| 64 |
sed -e "$sedcmd" "$tmpfile" | tr '\n' '\0' | xargs -0 -r sh -c 'rm -i -- "$0" "$@" </dev/tty' |
| 65 |
fi |
| 66 |
|
| 67 |
rm -- "$tmpfile" |
| 68 |
|
| 69 |
printf "Press any key to exit" |
| 70 |
read -r _ |