[pacman-dev] [PATCH v2 0/2] Correct package size calculation in makepkg
From: Ronan Pigott <rpigott@berkeley.edu> V2 implements suggestions by Allan to separate dirsize calculation into a util library function and implement a two-pass approach for hard linked and non hard linked files. Ronan Pigott (2): makepkg: use null-delimited file lists when tracking inodes makepkg: do not count hard linked file sizes multiple times configure.ac | 6 ++-- meson.build | 6 ++-- scripts/Makefile.am | 1 + scripts/libmakepkg/tidy/zipman.sh.in | 9 +++--- scripts/libmakepkg/util/dirsize.sh.in | 45 +++++++++++++++++++++++++++ scripts/libmakepkg/util/meson.build | 1 + scripts/makepkg.sh.in | 2 +- 7 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 scripts/libmakepkg/util/dirsize.sh.in -- 2.23.0
From: Ronan Pigott <rpigott@berkeley.edu> Signed-off-by: Ronan Pigott <rpigott@berkeley.edu> --- configure.ac | 6 +++--- meson.build | 6 +++--- scripts/libmakepkg/tidy/zipman.sh.in | 9 +++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/configure.ac b/configure.ac index d68b3591..fbb3b83b 100644 --- a/configure.ac +++ b/configure.ac @@ -370,18 +370,18 @@ GCC_VISIBILITY_CC # Host-dependant definitions DEFAULT_SEDINPLACEFLAGS=" --follow-symlinks -i" -INODECMD="stat -c '%i %n'" +INODECMD="stat -c '%i'" STRIP_BINARIES="--strip-all" STRIP_SHARED="--strip-unneeded" STRIP_STATIC="--strip-debug" case "${host_os}" in *bsd*) - INODECMD="stat -f '%i %N'" + INODECMD="stat -f '%i'" DEFAULT_SEDINPLACEFLAGS=" -i \"\"" ;; darwin*) host_os_darwin=yes - INODECMD="/usr/bin/stat -f '%i %N'" + INODECMD="/usr/bin/stat -f '%i'" DEFAULT_SEDINPLACEFLAGS=" -i ''" STRIP_BINARIES="" STRIP_SHARED="-S" diff --git a/meson.build b/meson.build index c5704efe..648dc80e 100644 --- a/meson.build +++ b/meson.build @@ -222,7 +222,7 @@ add_project_arguments('-include', 'config.h', language : 'c') filecmd = 'file' default_sedinplaceflags = ' --follow-symlinks -i' -inodecmd = 'stat -c \'%i %n\'' +inodecmd = 'stat -c \'%i\'' strip_binaries = '--strip-all' strip_shared = '--strip-unneeded' strip_static = '--strip-debug' @@ -236,13 +236,13 @@ endif os = host_machine.system() if os.startswith('darwin') - inodecmd = '/usr/bin/stat -f \'%i %n\'' + inodecmd = '/usr/bin/stat -f \'%i\'' default_sedinplaceflags = ' -i \'\'' strip_binaries = '' strip_shared = '-s' strip_static = '-s' elif os.contains('bsd') or os == 'dragonfly' - inodecmd = 'stat -f \'%i %n\'' + inodecmd = 'stat -f \'%i\'' default_sedinplaceflags = ' -i \'\'' endif diff --git a/scripts/libmakepkg/tidy/zipman.sh.in b/scripts/libmakepkg/tidy/zipman.sh.in index 3c2e261e..dd36ebef 100644 --- a/scripts/libmakepkg/tidy/zipman.sh.in +++ b/scripts/libmakepkg/tidy/zipman.sh.in @@ -33,9 +33,10 @@ tidy_modify+=('tidy_zipman') tidy_zipman() { if check_option "zipman" "y" && [[ -n ${MAN_DIRS[*]} ]]; then msg2 "$(gettext "Compressing man and info pages...")" - local file files inode link - while read -rd ' ' inode; do - read file + local file inode link + declare -A files + while read -rd $'\0' file; do + inode=$( @INODECMD@ "$file" ) find ${MAN_DIRS[@]} -type l 2>/dev/null | while read -r link ; do if [[ "${file}" -ef "${link}" ]] ; then @@ -56,6 +57,6 @@ tidy_zipman() { chmod 644 "${file}.gz" fi done < <(find ${MAN_DIRS[@]} -type f \! -name "*.gz" \! -name "*.bz2" \ - -exec @INODECMD@ '{}' + 2>/dev/null) + -print0) fi } -- 2.23.0
From: Ronan Pigott <rpigott@berkeley.edu> Signed-off-by: Ronan Pigott <rpigott@berkeley.edu> --- scripts/Makefile.am | 1 + scripts/libmakepkg/util/dirsize.sh.in | 45 +++++++++++++++++++++++++++ scripts/libmakepkg/util/meson.build | 1 + scripts/makepkg.sh.in | 2 +- 4 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 scripts/libmakepkg/util/dirsize.sh.in diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 88e9612d..63d09767 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -127,6 +127,7 @@ LIBMAKEPKG_IN = \ libmakepkg/util.sh \ libmakepkg/util/compress.sh \ libmakepkg/util/config.sh \ + libmakepkg/util/dirsize.sh \ libmakepkg/util/error.sh \ libmakepkg/util/message.sh \ libmakepkg/util/option.sh \ diff --git a/scripts/libmakepkg/util/dirsize.sh.in b/scripts/libmakepkg/util/dirsize.sh.in new file mode 100644 index 00000000..897464a1 --- /dev/null +++ b/scripts/libmakepkg/util/dirsize.sh.in @@ -0,0 +1,45 @@ +#!/usr/bin/bash +# +# dirsize.sh - calculate size of all files in a directory +# +# Copyright (c) 2019 Pacman Development Team <pacman-dev@archlinux.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +[[ -n "$LIBMAKEPKG_UTIL_DIRSIZE_SH" ]] && return +LIBMAKEPKG_UTIL_DIRSIZE_SH=1 + +LIBRARY=${LIBRARY:-'/usr/share/makepkg'} + + +dirsize() { + local inode size=0 + declare -A files + + while read -rd $'\0' file; do + size=$((size + $(wc -c < "$file"))) + done < <(find . -type f -links 1 -print0) + + # Size of unique hard-linked files + while read -rd $'\0' file; do + inode=$( @INODECMD@ "$file" ) + if [[ -z "${files[$inode]}" ]]; then + files[$inode]=$(wc -c < "$file") + size=$((size + ${files[$inode]})) + fi + done < <(find . -type f -links +1 -print0) + + echo $size +} diff --git a/scripts/libmakepkg/util/meson.build b/scripts/libmakepkg/util/meson.build index c29503b7..a160fd06 100644 --- a/scripts/libmakepkg/util/meson.build +++ b/scripts/libmakepkg/util/meson.build @@ -3,6 +3,7 @@ libmakepkg_module = 'util' sources = [ 'compress.sh.in', 'config.sh.in', + 'dirsize.sh.in', 'error.sh.in', 'message.sh.in', 'option.sh.in', diff --git a/scripts/makepkg.sh.in b/scripts/makepkg.sh.in index 997c8668..e75031c2 100644 --- a/scripts/makepkg.sh.in +++ b/scripts/makepkg.sh.in @@ -584,7 +584,7 @@ write_kv_pair() { } write_pkginfo() { - local size="$(find . -type f -exec cat {} + 2>/dev/null | wc -c)" + local size=$(dirsize) merge_arch_attrs -- 2.23.0
participants (1)
-
Ronan Pigott