[pacman-dev] [PATCH v3] Add testrunner for makepkg-template

[Florian Pritz]

Signed-off-by: Florian Pritz <bluewind at xinu.at>
---

v3:
 + suggested by Andrew
  - take the executable as $1 rather than just the dir
  - force C locale
  - use $testdir when normalizing paths
  - fix output line of successful testcase
  - remove useless begin comment

 + replace $testdir more than once per line (g modifier)
 + use perl instead of sed and use quotemeta to do simple string replacements
   without interpreting the variable as a regex
 + reorder diff arguments so --- points to expected output and +++ to actual

On 22.12.2014 05:19, Andrew Gregory wrote:
>> +		-e 's| at '$scriptdir'/makepkg-template line [0-9]\+\.$||'
> 
> makepkg-template should be updated to not include line numbers in
> error messages rather than trying to filter them out.

I find line numbers rather helpful when debugging (which is also why I keep
output-full around even though I currently provide no way to keep the tempdir),
but I agree it's not the cleanest check.

> 
>> +	printf "exitcode: %s\n" "$exitcode" >> "$TMPDIR/$testcase/output"
> 
> It would be better if we could test the exit code, output, and result
> independently.

result and output are already tested independently and adding the exit code to
the output means I need less files per test case. Since I sometimes expect exit
codes != 0 I need a way to check for a specific one and this seemed easiest.

Would you want a dedicated file with just one number inside or maybe some kind
of "config" file per testcase? That could also store additional arguments
(currently "arguments" file, used only once I believe). It would add an
additional file to every test case though.

>> +
>> +	if [[ -f "$TMPDIR/$testcase/result" ]]; then
>> +		if ! diff -u "$TMPDIR/$testcase/result" "$testdir/$testcase/expected_result"; then
> 
> The diff output needs to be marked as diagnostic output.  I'm working
> on a TAP generator for bash that we can use that will do that for us
> as well as taking care of all the counting.

Any ETA? I have not yet changed that. That said the test suite seems to put
this output into the log file just fine AFAICT.

>> +	else
>> +		(( ++fail ))
>> +		printf "not ok %d - %s\n" "$testcount" "$input"
>> +		printf '# [TEST %3s]: FAIL\n' "$testcount"
> 
> Redundant failure message.

I copied that from human_to_size_test.sh. I have no idea how the testsuite
actually works and why this was there initially.

 Makefile.am                           |  2 +
 test/scripts/Makefile.am              |  1 +
 test/scripts/makepkg-template_test.sh | 91 +++++++++++++++++++++++++++++++++++
 3 files changed, 94 insertions(+)
 create mode 100755 test/scripts/makepkg-template_test.sh

diff --git a/Makefile.am b/Makefile.am
index e9b3dfa..ab669ef 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -28,6 +28,7 @@ $(top_srcdir)/test/pacman/tests/TESTS: $(wildcard test/pacman/tests/*.py*)
 
 TESTS =  test/scripts/parseopts_test.sh \
 				 test/scripts/human_to_size_test.sh \
+				 test/scripts/makepkg-template_test.sh \
 				 test/scripts/pacman-db-upgrade-v9.py \
 				 test/util/pacsorttest.sh \
 				 test/util/vercmptest.sh
@@ -37,6 +38,7 @@ TEST_SUITE_LOG = test/test-suite.log
 TEST_EXTENSIONS = .py
 AM_TESTS_ENVIRONMENT = \
 	PMTEST_UTIL_DIR=$(top_builddir)/src/util/; export PMTEST_UTIL_DIR; \
+	PMTEST_SCRIPT_DIR=$(top_srcdir)/scripts/; export PMTEST_SCRIPT_DIR; \
 	PMTEST_SCRIPTLIB_DIR=$(top_srcdir)/scripts/library/; export PMTEST_SCRIPTLIB_DIR;
 LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) \
 								 $(top_srcdir)/build-aux/tap-driver.sh
diff --git a/test/scripts/Makefile.am b/test/scripts/Makefile.am
index 8d6bc84..ed1a951 100644
--- a/test/scripts/Makefile.am
+++ b/test/scripts/Makefile.am
@@ -1,6 +1,7 @@
 check_SCRIPTS = \
 	parseopts_test.sh \
 	pacman-db-upgrade-v9.py \
+	makepkg-template_test.sh \
 	human_to_size_test.sh
 
 noinst_SCRIPTS = $(check_SCRIPTS)
diff --git a/test/scripts/makepkg-template_test.sh b/test/scripts/makepkg-template_test.sh
new file mode 100755
index 0000000..46f5929
--- /dev/null
+++ b/test/scripts/makepkg-template_test.sh
@@ -0,0 +1,91 @@
+#!/bin/bash
+
+declare -i testcount=0 fail=0 pass=0 total=0
+
+# source the library function
+script=${1:-${PMTEST_SCRIPT_DIR}/makepkg-template}
+if [[ -z $script || ! -f $script ]]; then
+	printf "Bail out! makepkg-template executable (%s) could not be located\n" "${script}"
+	exit 1
+fi
+
+testdir="${0%/*}/makepkg-template-tests"
+
+total=$(find "$testdir" -maxdepth 1 -mindepth 1 -type d | wc -l)
+
+run_test() {
+	local testcase=$1 exitcode
+	local -i failed=0
+
+	(( ++testcount ))
+
+	local -a arguments
+
+	if [[ -f "$testdir/$testcase/arguments" ]]; then
+		arguments=($(cat "$testdir/$testcase/arguments"))
+	fi
+
+	mkdir "$TMPDIR/$testcase"
+
+	LC_ALL=C "$script" \
+		--template-dir "$testdir/$testcase/templates" \
+		-p "$testdir/$testcase/PKGBUILD" \
+		-o "$TMPDIR/$testcase/result" \
+		&> "$TMPDIR/$testcase/output_full" "${arguments[@]}"
+	exitcode=$?
+
+	perl -p <"$TMPDIR/$testcase/output_full" >"$TMPDIR/$testcase/output" \
+		-e 's| at \Q'"$script"'\E line [0-9]+\.$||;' \
+
+	if [[ "${testdir%/*}" != "." ]]; then
+		perl -pi -e 's|\Q'"${testdir%/*}"'\E|.|g;' "$TMPDIR/$testcase/output"
+	fi
+
+	printf "exitcode: %s\n" "$exitcode" >> "$TMPDIR/$testcase/output"
+
+	if [[ -f "$TMPDIR/$testcase/result" ]]; then
+		if ! diff -u "$testdir/$testcase/expected_result" "$TMPDIR/$testcase/result"; then
+			failed=1
+		fi
+	else
+		if [[ -f "$testdir/$testcase/expected_result" ]]; then
+			echo "$testcase: Found expected_result file but no result file"
+			failed=1
+		fi
+	fi
+
+	if ! diff -u "$testdir/$testcase/expected_output" "$TMPDIR/$testcase/output"; then
+		failed=1
+	fi
+
+	if ((!failed)); then
+		(( ++pass ))
+		printf "ok %d - %s\n" "$testcount" "$testcase"
+	else
+		(( ++fail ))
+		printf "not ok %d - %s\n" "$testcount" "$input"
+		printf '# [TEST %3s]: FAIL\n' "$testcount"
+	fi
+}
+
+summarize() {
+	rm -rf "$TMPDIR"
+
+	if (( !fail )); then
+		printf '# All %s tests successful\n\n' "$testcount"
+		exit 0
+	else
+		printf '# %s of %s tests failed\n\n' "$fail" "$testcount"
+		exit 1
+	fi
+}
+TMPDIR="$(mktemp -d "/tmp/${0##*/}.XXXXXX")"
+trap 'summarize' EXIT
+
+echo "1..$total"
+
+for dir in "$testdir/"*; do
+	if [[ -d "$dir" ]]; then
+		run_test "${dir##*/}"
+	fi
+done
-- 
2.2.1