dev: errors: improve error test generation

hledger/test/errors/Makefile

@@ -1,51 +1,28 @@
 HLEDGER ?= hledger
 
+ERRORSCRIPTS=*.j *.timeclock # *.timedot *.csv
+
 # Check error messages of hledger in $PATH against current error tests.
 test:
 	@printf "Running error message tests with hledger $$($(HLEDGER) --version | awk '{print $$2}'):\n"
 	shelltest -w $(HLEDGER) *.test
 
-TESTJOURNALS=*.j
-
 # Update error message tests and readme based on the latest test journals
 # and error output of hledger in $PATH.
 update: tests readme
 
 tests:
-	@printf "Updating *.test with the error messages of hledger $$(hledger --version | awk '{print $$2}')\n"
+	@printf "(Re)generating *.test with the error messages of hledger $$($(HLEDGER) --version | awk '{print $$2}')\n"
 	@read -p "ok ? Press enter: "
-	for f in $(TESTJOURNALS); do make -s $$(basename $$f .j).test; done
+	@for f in $(ERRORSCRIPTS); do echo "HLEDGER=$(HLEDGER) ./hledger2shelltest $$f"; HLEDGER=$(HLEDGER) ./hledger2shelltest $$f; done
-	@printf "\nNow please edit *.test and limit regexps to 300 chars to avoid shelltestrunner limitation\n\n"
 
-# Generate a shelltest. Run the test script/journal to generate the error message.
+readme: $(ERRORSCRIPTS)
-# Since the error will contain an absolute file path, we must:
+	@printf "Updating README.md with the error messages of hledger $$($(HLEDGER) --version)\n"
-# 1. remove most of the file path
-# 2. test with a (multiline) regex rather than literal text
-# 3. backslash-quote most forward slashes in error messages
-# 4. neutralise any remaining problematic error text (eg in parseable-regexps.test)
-%.test: %.j
-	head -1 $< | sed -E 's%#!/usr/bin/env -S (.*)%$$$$$$ \1 $<%' >$@
-	printf ">>>2 /" >>$@
-	-./$< 2>&1 | sed -E \
-		-e 's%(hledger: Error: ).*/\./(.*)%\1.*\2%' \
-		-e 's%/%\\/%g' \
-		-e 's/\^/\\^/g' \
-		-e 's/\$$/\\$$/g' \
-		-e 's/\(/\\(/g' \
-		-e 's/\)/\\)/g' \
-		-e 's/\|/\\|/g' \
-		>>$@
-	printf "/\n>>>= 1" >>$@
-#		-e 's%alias \\/\(\\/%alias \\/\\(\\/%' \
-#		-e 's%compiled: \(%compiled: \\(%' \
-
-readme: $(TESTJOURNALS)
-	@printf "Updating README.md with the error messages of hledger $$(hledger --version | awk '{print $$2}')\n"
 	@read -p "ok ? Press enter: "
 	sed '/<!-- GENERATED: -->/q' <README.md >README.md.tmp
-	echo "$$(hledger --version | cut -d, -f1) error messages:" >>README.md.tmp
+	echo "$$($(HLEDGER) --version | cut -d, -f1) error messages:" >>README.md.tmp
-	for f in $(TESTJOURNALS); do \
+	for f in $(ERRORSCRIPTS); do \
-		printf '\n### %s\n```\n%s\n```\n\n' "$$(basename "$$f" .j)" "$$(./"$$f" 2>&1)"; \
+		printf '\n### %s\n```\n%s\n```\n\n' "$$(echo "$$f" | sed -E 's/\.[^.]+$$//')" "$$(./"$$f" 2>&1)"; \
 	done >>README.md.tmp
 	mv README.md.tmp README.md
 

hledger/test/errors/hledger2shelltest

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# hledger2shelltest SCRIPT
+#
+# Speaking generally: given an executable hashbang script (beginning with #!/usr/bin/env),
+# this generates a similarly-named shelltestrunner test that will repeatably
+# run the same command as the script and test its (stderr) output.
+# (Ideally, this would be built in to shelltestrunner.)
+# More precisely, this generates a test expecting no stdout, the given stderr,
+# and an error exit code, for scripts reproducing various hledger errors.
+#
+# The script is run once to capture its output, which is then adjusted
+# for use in a shelltest regex matcher:
+# - common regex metacharacters are escaped
+# - file paths are simplified
+# - any remaining problematic text is sanitised
+# - the regex is trimmed to <= 300 chars, to avoid a shelltestrunner limitation.
+
+SCRIPT="$1"
+TEST=$(echo "$SCRIPT" | sed -E 's/\.[^.]+$//').test
+
+{
+head -1 "$SCRIPT" | sed -E "s%#!/usr/bin/env -S (.*)%\$\$\$ \1 $SCRIPT%"
+printf ">>>2 /"
+"./$SCRIPT" 2>&1 | sed -E \
+  -e 's%(hledger: Error: ).*/\./(.*)%\1.*\2%' \
+  -e 's%/%\\/%g' \
+  -e 's/\^/\\^/g' \
+  -e 's/\$/\\$/g' \
+  -e 's/\(/\\(/g' \
+  -e 's/\)/\\)/g' \
+  -e 's/\|/\\|/g' \
+  | head -c 300
+printf "/\n>>>= 1\n"
+} >"$TEST"
+
+
+#  -e 's%alias \\/\(\\/%alias \\/\\(\\/%' \
+#  -e 's%compiled: \(%compiled: \\(%' \
+
+# gnused() {  # GNU sed, called gsed on mac
+#     if hash gsed 2>/dev/null; then gsed "$@"; else sed "$@"; fi
+# }