[PATCH olang 1/2] tests: add comment based integration tests mechanism

[Carlos Maniero <carlos@maniero.me>]

To create an integration tests you just need to create a olang file at
*tests/integration/tests* directory. The assertions are performed
thought a couple of comments.

All tests features must start with the TEST statement. Arguments can be
provided to the test feature thought invocation using
*test_feature(arg_name_1=arg_value_1,arg_name_n=arg_value_n)* and
multi-line text (test contents) can be passed thought tests over the
WITH statement.

Available test features:
========================

test_compile
------------

Compiles the program, assert the exit code and the stdout/stderr if WITH
statement is present.

test_run_binary
---------------

Run the binary, assert the exit code and check the binary stdout/stderr
if WITH statement is present. The test_run_binary depends on the
test_compile.

test_ast
--------

Executes the *olang --dump-ast* over the test file and check if the ast
matches for the given test contents over the diff command.

test_contains_tokens
--------------------

Validate that every given token by the test contents is present on the
output of *olang --dump-tokens*. Since the language has the line_feed
token it is impossible to test the EOF once when added a comment the EOF
will be located at line + 1. So that the tokens test the first n-lines
defined. So instead of performing a diff like test_ast it validates line
by line.

Signed-off-by: Carlos Maniero <carlos@maniero.me>
---
 Makefile                                  |   3 -
 tests/integration/Makefile                |  34 +---
 tests/integration/test.sh                 | 231 ++++++++++++++++++++++
 tests/integration/tests/0001_main_exit.ol |  46 +++++
 4 files changed, 283 insertions(+), 31 deletions(-)
 create mode 100755 tests/integration/test.sh
 create mode 100644 tests/integration/tests/0001_main_exit.ol

diff --git a/Makefile b/Makefile
index 27337d4..2f5273b 100644
--- a/Makefile
+++ b/Makefile
@@ -19,13 +19,11 @@ $(BUILD_DIR):
 .PHONY: format
 format: $(SRCS) $(HEADERS)
 	clang-format --dry-run --Werror $?
-	$(MAKE) -C tests/integration/ format
 	$(MAKE) -C tests/unit/ format
 
 .PHONY: format-fix
 format-fix: $(SRCS) $(HEADERS)
 	clang-format -i $?
-	$(MAKE) -C tests/integration/ format-fix
 	$(MAKE) -C tests/unit/ format-fix
 
 .PHONY: integration-test
@@ -40,7 +38,6 @@ unit-test:
 
 .PHONY: clean
 clean:
-	$(MAKE) -C tests/integration/ clean
 	$(MAKE) -C tests/unit/ clean
 	@rm -rf build/ $(TARGET)
 
diff --git a/tests/integration/Makefile b/tests/integration/Makefile
index 4625707..bda51af 100644
--- a/tests/integration/Makefile
+++ b/tests/integration/Makefile
@@ -1,31 +1,9 @@
-SRCS        := $(wildcard *_test.c)
-OBJS        := $(patsubst %_test.c, %_test.o, $(SRCS))
-CFLAGS      := -I../../src -I../shared
-TESTS       := $(patsubst %_test.c, %_test, $(SRCS))
-EXEC_TESTS  := $(patsubst %_test, ./%_test, $(TESTS))
-MUNIT_SRC   := ../shared/munit.c
-MUNIT       := ./munit.o
+TESTER_SRC := ./test.sh
+TESTS := $(wildcard ./tests/*.ol)
 
 .PHONY: all
-all: $(MUNIT) proc_exec.o cli_runner.o $(TESTS)
-	@for file in $(EXEC_TESTS); do \
-                ./"$$file"; \
+all:
+	@set -e; \
+	for file in $(TESTS); do \
+                $(TESTER_SRC) "$$file"; \
         done
-
-.PHONY: clean
-clean:
-	$(RM) *.o *_test
-
-.PHONY: format
-format: $(SRCS)
-	clang-format --dry-run --Werror $?
-
-.PHONY: format-fix
-format-fix: $(SRCS)
-	clang-format -i $?
-
-cli_test: $(MUNIT) proc_exec.o cli_runner.o cli_test.o
-	$(CC) $? $(CFLAGS) -o $@
-
-$(MUNIT):
-	$(CC) -c $(MUNIT_SRC) $(CFLAGS) -o $(MUNIT)
diff --git a/tests/integration/test.sh b/tests/integration/test.sh
new file mode 100755
index 0000000..ae0a707
--- /dev/null
+++ b/tests/integration/test.sh
@@ -0,0 +1,231 @@
+#!/bin/sh
+# Copyright (C) 2024 olang mantainers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+# Ignores variables been modified in a subshell
+# shellcheck disable=SC2030,SC2031
+OLANG_PATH="../../olang"
+TEST_FILE="$1"
+
+TEST_TMP_FILES="$TEST_FILE.test"
+TEST_TMP_BIN="$TEST_TMP_FILES.bin"
+
+# Execution state vars
+TEST_NAME=""
+TEST_INVOCATION=""
+TEST_LINE_NUMBER=""
+TEST_CONTENTS_PATH=""
+TEST_ARGS=""
+TEST_SOME_PASSED=""
+
+# UI
+COLOR_RED=1
+COLOR_GREEN=2
+COLOR_YELLOW=3
+COLOR_CYAN=6
+COLOR_GRAY=7
+
+colored() {
+  text="$1"
+
+  if [ -t 1 ]; then
+    if tput setaf 1 > /dev/null 2>&1; then
+      color=$(tput setaf "$2")
+      reset=$(tput sgr0)
+
+      printf "%s%s%s" "$color" "$text" "$reset"
+      return
+    fi
+  fi
+
+  printf "%s" "$text"
+}
+# end UI
+
+# test output
+print_failed() {
+  reason="$1"
+
+  if [ -n "$TEST_SOME_PASSED" ]; then
+    echo
+  fi
+
+  colored "$TEST_FILE:$TEST_LINE_NUMBER:1: " $COLOR_GRAY 
+  colored "[$TEST_INVOCATION] " $COLOR_CYAN
+  colored "failed" $COLOR_RED
+  if [ -n "$reason" ]; then
+    printf ": %s" "$reason"
+  fi
+  echo
+}
+
+print_passed() {
+  colored "." $COLOR_GREEN
+  TEST_SOME_PASSED="true"
+}
+# end test output
+
+# expectations
+expect_output_contains() {
+  actual_file="$1"
+  expected_file="$2"
+  original_line_number="$TEST_LINE_NUMBER"
+
+  while read -r expected_line ; do
+    TEST_LINE_NUMBER=$((TEST_LINE_NUMBER+1))
+
+    if [ "$(grep "$(printf "%s" "$expected_line" | sed 's/\\/\\\\/g')" "$actual_file" | wc -c)" = "0" ]; then
+      print_failed
+      colored "(not found) $expected_line" $COLOR_YELLOW
+      echo
+      colored "$(awk '{print "(actual) " $0}' "$actual_file")" $COLOR_GRAY
+      echo
+      exit 1
+    fi
+  done < "$expected_file"
+
+  TEST_LINE_NUMBER="$original_line_number"
+}
+
+diff_output() {
+  actual_file="$1"
+  expected_file="$2"
+
+  if cmp -s "$expected_file" "$actual_file"; then
+    return
+  fi
+
+  print_failed "match failed"
+  diff "$actual_file" "$expected_file" -u --color
+  exit 1
+}
+
+cleanup() {
+  rm -rf "$TEST_TMP_FILES"*
+}
+
+main() {
+  all_test_end="$(grep -n "# END" "$TEST_FILE" | awk -F: '{print $1}')"
+
+  grep -n "# TEST " "$TEST_FILE" | while read -r line ; do
+    TEST_LINE_NUMBER="$(echo "$line" | awk -F: '{ print $1 }')"
+    TEST_INVOCATION="$(echo "$line" | awk -F: '{ print $2 }' | sed 's/^\# TEST //' | sed 's/ WITH$//')"
+
+    TEST_NAME="$(echo "$TEST_INVOCATION" | awk -F\( '{ print $1 }')"
+    TEST_ARGS="$(echo "$TEST_INVOCATION" | awk -F\( '{ print $2 }' | sed 's/)$//')"
+
+    has_with_contents="$(echo "$line" | sed 's/.*WITH$/true/')"
+    end_line="$TEST_LINE_NUMBER"
+
+    if [ "$has_with_contents" = "true" ]; then
+      for test_end in $all_test_end; do
+        if [ "$test_end" -gt "$TEST_LINE_NUMBER" ]; then
+          end_line="$test_end"
+          break
+        fi
+      done
+
+      TEST_CONTENTS_PATH="$TEST_TMP_FILES.$TEST_LINE_NUMBER.$end_line.partial"
+
+      awk -v start="$TEST_LINE_NUMBER" -v end="$end_line" 'NR > start && NR < end' "$TEST_FILE" \
+        | sed 's/^\# /#/' | sed 's/^\#//' > "$TEST_CONTENTS_PATH"
+    else
+      TEST_CONTENTS_PATH=""
+    fi
+
+    if type "$TEST_NAME" | grep -q 'function'; then
+        "$TEST_NAME"
+        print_passed
+    else
+        print_failed "test not implemented"
+    fi
+  done || exit 1;
+
+  cleanup
+  echo
+}
+
+get_test_args() {
+  arg_name="$1"
+  echo "$TEST_ARGS" | sed "s/,/\n/g" | grep "$arg_name=" | sed "s/^$arg_name=//"
+}
+
+assert_contents_path() {
+  if [ -z "$TEST_CONTENTS_PATH" ]; then
+    print_failed "missing WITH block for test contents"
+    exit 5
+  fi
+}
+
+# Test functions
+test_compile() {
+  expected_exit_code="$(get_test_args "exit_code")"
+  actual_output_file="$TEST_TMP_FILES.$TEST_LINE_NUMBER.compiler_output"
+
+  $OLANG_PATH "$TEST_FILE" -o "$TEST_TMP_BIN" > "$actual_output_file" 2>&1
+  exit_code="$?"
+
+  if [ -n "$expected_exit_code" ]; then
+    if [ "$expected_exit_code" -ne "$exit_code" ]; then
+      print_failed "expected compiler exit code: $expected_exit_code actual: $exit_code"
+      exit 1
+    fi
+  fi
+
+  if [ -n "$TEST_CONTENTS_PATH" ]; then
+    diff_output "$actual_output_file" "$TEST_CONTENTS_PATH"
+  fi
+}
+
+test_ast() {
+  assert_contents_path
+
+  actual_output_file="$TEST_TMP_FILES.$TEST_LINE_NUMBER.ast_output"
+
+  $OLANG_PATH "$TEST_FILE" --dump-ast > "$actual_output_file" 2>&1
+
+  diff_output "$actual_output_file" "$TEST_CONTENTS_PATH"
+}
+
+test_contains_tokens() {
+  assert_contents_path
+
+  actual_output_file="$TEST_TMP_FILES.$TEST_LINE_NUMBER.tokens_output"
+
+  $OLANG_PATH "$TEST_FILE" --dump-tokens > "$actual_output_file" 2>&1
+
+  expect_output_contains "$actual_output_file" "$TEST_CONTENTS_PATH"
+}
+
+test_run_binary() {
+  expected_exit_code="$(get_test_args "exit_code")"
+  actual_output_file="$TEST_TMP_FILES.$TEST_LINE_NUMBER.run_output"
+
+  "$TEST_TMP_BIN" > "$actual_output_file" 2>&1
+  exit_code="$?"
+
+  if [ -n "$expected_exit_code" ]; then
+    if [ "$expected_exit_code" -ne "$exit_code" ]; then
+      print_failed "expected compiler exit code: $expected_exit_code actual: $exit_code"
+      exit 1
+    fi
+  fi
+
+  if [ -n "$TEST_CONTENTS_PATH" ]; then
+    diff_output "$actual_output_file" "$TEST_CONTENTS_PATH"
+  fi
+}
+
+main
diff --git a/tests/integration/tests/0001_main_exit.ol b/tests/integration/tests/0001_main_exit.ol
new file mode 100644
index 0000000..f446eb9
--- /dev/null
+++ b/tests/integration/tests/0001_main_exit.ol
@@ -0,0 +1,46 @@
+# Copyright (C) 2024 olang mantainers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+# A minimal olang program
+fn main(): u32 {
+  return 0
+}
+
+# TEST test_compile(exit_code=0)
+#
+# TEST test_run_binary(exit_code=0)
+#
+# TEST test_ast WITH
+# Translation_Unit
+# `-Function_Definition <name:main> <return:0>
+#   `-Block
+#     `-Return_Statement
+#       `-Literal <kind:u32> <value:0>
+# END
+#
+# TEST test_contains_tokens WITH
+# ./tests/0001_main_exit.ol:17:1: <fn>
+# ./tests/0001_main_exit.ol:17:4: <identifier>
+# ./tests/0001_main_exit.ol:17:8: <(>
+# ./tests/0001_main_exit.ol:17:9: <)>
+# ./tests/0001_main_exit.ol:17:10: <:>
+# ./tests/0001_main_exit.ol:17:12: <identifier>
+# ./tests/0001_main_exit.ol:17:16: <{>
+# ./tests/0001_main_exit.ol:17:17: <line_feed>
+# ./tests/0001_main_exit.ol:18:3: <return>
+# ./tests/0001_main_exit.ol:18:10: <number>
+# ./tests/0001_main_exit.ol:18:11: <line_feed>
+# ./tests/0001_main_exit.ol:19:1: <}>
+# END
-- 
2.34.1