tests: implement a better integration test framework than shellspec

(for our usecase at least)

.gitignore

@@ -33,3 +33,4 @@ bash.txt
 .cache/
 compile_commands.json
 .vscode
+report.txt

.shellspec

@@ -1,12 +0,0 @@
---require spec_helper
-
-## Default kcov (coverage) options
-# --kcov-options "--include-path=. --path-strip-level=1"
-# --kcov-options "--include-pattern=.sh"
-# --kcov-options "--exclude-pattern=/.shellspec,/spec/,/coverage/,/report/"
-
-## Example: Include script "myprog" with no extension
-# --kcov-options "--include-pattern=.sh,myprog"
-
-## Example: Only specified files/directories
-# --kcov-options "--include-pattern=myprog,/lib/"

Makefile

@@ -82,7 +82,8 @@ minishell_objs = $(addsuffix .o,src/$(NAME)) $(objs)
 all_objs = $(minishell_objs)
 deps = $(all_objs:.o=.d)
 
-.PHONY: all clean fclean re norm tests ctests ctestsa ctestsub ctestst alla allub allt
+.PHONY: all clean fclean re norm tests ctests ctestsa ctestsub ctestst alla \
+	allub allt testsa testsub testst inttests inttestsa inttestsub inttestst
 
 all: $(NAME)
 
@@ -116,24 +117,48 @@ norm:
 	norminette src libft | grep -v OK || true
 
 tests:
-	@echo "Running tests with AddressSanitizer..."
-	+CFLAGS="$(CFLAGS) $(ASAN)" make re
-	shellspec
-	+make ctestsa
-
-	@echo "Running tests with UndefinedBehaviourSanitizer..."
-	+CFLAGS="$(CFLAGS) $(UBSAN)" make re
-	shellspec
-	+make ctestsub
-
-	@echo "Running tests with ThreadSanitizer..."
-	+CFLAGS="$(CFLAGS) $(TSAN)" make re
-	shellspec
-	+make ctestst
-
+	+make fclean
+	+make testsa
+	+make fclean
+	+make testsub
+	+make fclean
+	+make testst
 	@echo "All tests passed!"
 
-ctests: $(LIBFT)
+testsa: CFLAGS += $(ASAN)
+testsa:
+	@echo "Running tests with AddressSanitizer..."
+	+make inttests
+	+make ctests
+
+testsub: CFLAGS += $(UBSAN)
+testsub: all
+	@echo "Running tests with UndefinedBehaviourSanitizer..."
+	+make inttests
+	+make ctests
+
+testst: CFLAGS += $(TSAN)
+testst: all
+	@echo "Running tests with ThreadSanitizer..."
+	+make inttests
+	+make ctests
+
+inttests: all
+	./test.sh
+
+inttestsa: CFLAGS += $(ASAN)
+inttestsa: all
+	./test.sh
+
+inttestsub: CFLAGS += $(UBSAN)
+inttestsub: all
+	./test.sh
+
+inttestst: CFLAGS += $(TSAN)
+inttestst: all
+	./test.sh
+
+ctests: $(LIBFT) all
 	+make -C tests
 
 ctestsa: CFLAGS += $(ASAN)

spec/minishell_spec.sh

@@ -1,22 +0,0 @@
-It "parses null for empty command"
-  Data
-    #|
-  End
-
-  When call ./minishell
-  The output should eq ""
-End
-
-It "a single command is parsed"
-  Data
-    #|echo hello
-  End
-
-  When call ./minishell
-  The output should eq \
-" ╰─ t_cmdgroup
-     ├─ t_cmdlist
-     │   ├─ num_cmds = 1
-     │   ╰─ cmd[0]
-     ╰─ t_redir_list"
-End

spec/spec_helper.sh

@@ -1,24 +0,0 @@
-# shellcheck shell=sh
-
-# Defining variables and functions here will affect all specfiles.
-# Change shell options inside a function may cause different behavior,
-# so it is better to set them here.
-# set -eu
-
-# This callback function will be invoked only once before loading specfiles.
-spec_helper_precheck() {
-  # Available functions: info, warn, error, abort, setenv, unsetenv
-  # Available variables: VERSION, SHELL_TYPE, SHELL_VERSION
-  : minimum_version "0.28.1"
-}
-
-# This callback function will be invoked after a specfile has been loaded.
-spec_helper_loaded() {
-  :
-}
-
-# This callback function will be invoked after core modules has been loaded.
-spec_helper_configure() {
-  # Available functions: import, before_each, after_each, before_all, after_all
-  : import 'support/custom_matcher'
-}

test.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+declare -i FAILED=0
+declare -i SUCCEDED=0
+declare -i RAN=0
+NAME=""
+
+failed_report()
+{
+	echo "++++++++++++++++++++++++++++++++++++++++++++++++++++"
+	echo $NAME
+	echo "++++++++++++++++++++++++++++++++++++++++++++++++++++"
+	echo "++++ Failed test:"
+	cat /tmp/input.minishell
+	echo "++++ Got output:"
+	cat /tmp/got.minishell
+	echo "++++ But expected:"
+	cat /tmp/expected.minishell
+	echo "++++ Diff:"
+	echo "(red is got, green is expected)"
+	diff --color=always /tmp/got.minishell /tmp/expected.minishell
+	echo "++++ Run this to debug:"
+	echo "echo '$(cat /tmp/input.minishell)' | ./minishell"
+}
+
+failed()
+{
+	failed_report >> report.txt
+	echo -n "F"
+	FAILED+=1
+}
+
+succeded()
+{
+	echo -n "."
+	SUCCEDED+=1
+}
+
+assert()
+{
+	diff -q /tmp/got.minishell /tmp/expected.minishell > /dev/null && succeded || failed
+}
+
+when_run()
+{
+	cat > /tmp/input.minishell
+	./minishell &> /tmp/got.minishell < /tmp/input.minishell
+	RAN+=1
+	NAME=$1
+}
+
+expecting()
+{
+	cat > /tmp/expected.minishell
+	assert
+}
+
+setup()
+{
+	rm -f report.txt
+}
+
+finalize()
+{
+	echo
+	echo "$RAN tests ran. $SUCCEDED succeded, $FAILED failed."
+	if [ $SUCCEDED -eq $RAN ];
+	then
+		echo "All integration tests passed!"
+	else
+		echo "cat report.txt"
+		echo "to see details of failed tests"
+	fi
+}
+
+setup
+
+when_run <<EOF "empty inputs returns nothing"
+
+EOF
+expecting <<EOF
+EOF
+
+when_run <<EOF "simple command is parsed"
+echo hello
+EOF
+expecting <<EOF
+ ╰─ t_cmdgroup
+     ├─ t_cmdlist
+     │   ├─ num_cmds = 1
+     │   ╰─ cmd[0]
+     │       ├─ t_cmdlist_item
+     │       │   ╰─ t_pipeline
+     │       │       ├─ num_cmd = 1
+     │       │       ╰─ cmd[0]
+     │       │           ╰─ t_simple_cmd
+     │       │               ├─ words = [echo][hello]
+     │       │               ╰─ (no redirections)
+     │       ╰─ t_operator = END
+     ╰─ (no redirections)
+EOF
+
+finalize