[PATCH olang 1/4] tests: add comment based integration tests mechanism

[Carlos Maniero <carlos@maniero.me>]

The old munit based integration tests was replaced by a new mechanism
that makes testing more tasteful and easy to perform.

To create an integration tests you just need to create a olang file at
*tests/integration/tests* directory. The assertions are performed
thought a couple of comments.

spec: some text to describe what are you testing
compiler_exit: the compilers exit code
compiler_output: the compilers output, %empty% for empty output
program_exit: the program exit code
ast: the ast output
tokens: the tokens

Since the language has the line_feed token it is impossible to test the
EOF once when added a comment the EOF will be located at line + 1. So
that the tokens test the first n-lines defined. The AST compares the
entire AST tough.

Signed-off-by: Carlos Maniero <carlos@maniero.me>
---
 Makefile                                  |   1 -
 tests/integration/Makefile                |  33 +---
 tests/integration/test.sh                 | 182 ++++++++++++++++++++++
 tests/integration/tests/0001_main_exit.ol |  27 ++++
 4 files changed, 217 insertions(+), 26 deletions(-)
 create mode 100755 tests/integration/test.sh
 create mode 100644 tests/integration/tests/0001_main_exit.ol

diff --git a/Makefile b/Makefile
index cdfc8e1..a0130f5 100644
--- a/Makefile
+++ b/Makefile
@@ -19,7 +19,6 @@ $(BUILD_DIR):
 .PHONY: linter
 linter: $(SRCS) $(HEADERS)
 	clang-format --dry-run --Werror $?
-	$(MAKE) -C tests/integration/ linter
 	$(MAKE) -C tests/unit/ linter
 
 .PHONY: linter-fix
diff --git a/tests/integration/Makefile b/tests/integration/Makefile
index db2b7d9..567ce88 100644
--- a/tests/integration/Makefile
+++ b/tests/integration/Makefile
@@ -1,31 +1,14 @@
-SRCS        := $(wildcard *_test.c)
-OBJS        := $(patsubst %_test.c, %_test.o, $(SRCS))
-CFLAGS      := -I../../src -I../shared
-TESTS       := $(patsubst %_test.c, %_test, $(SRCS))
-EXEC_TESTS  := $(patsubst %_test, ./%_test, $(TESTS))
-MUNIT_SRC   := ../shared/munit.c
-MUNIT       := ./munit.o
+TESTER_SRC := ./test.sh
+TESTS := $(wildcard ./tests/*.ol)
 
 .PHONY: all
-all: $(MUNIT) proc_exec.o cli_runner.o $(TESTS)
-	@for file in $(EXEC_TESTS); do \
-                ./"$$file"; \
+all:
+	@set -e; \
+	for file in $(TESTS); do \
+                $(TESTER_SRC) "$$file"; \
+		echo; \
         done
 
 .PHONY: clean
 clean:
-	$(RM) *.o *_test
-
-.PHONY: linter
-linter: $(SRCS)
-	clang-format --dry-run --Werror $?
-
-.PHONY: linter-fix
-linter-fix: $(SRCS)
-	clang-format -i $?
-
-cli_test: $(MUNIT) proc_exec.o cli_runner.o cli_test.o
-	$(CC) $? $(CFLAGS) -o $@
-
-$(MUNIT):
-	$(CC) -c $(MUNIT_SRC) $(CFLAGS) -o $(MUNIT)
+	@rm -rf ./tests/*.ol.test*
diff --git a/tests/integration/test.sh b/tests/integration/test.sh
new file mode 100755
index 0000000..6ab03de
--- /dev/null
+++ b/tests/integration/test.sh
@@ -0,0 +1,182 @@
+COMPILER_PATH="../../olang"
+TEST_FILE="$1"
+
+PROGRAM="$TEST_FILE.test"
+
+COMPILER_OUTPUT_FILE="$TEST_FILE.test.compiler_output"
+COMPILER_EXPECT_OUTPUT_FILE="$TEST_FILE.test.expected_compiler_output"
+
+AST_OUTPUT_FILE="$TEST_FILE.test.ast_output"
+AST_EXPECT_OUTPUT_FILE="$TEST_FILE.test.expected_ast_output"
+
+TOKENS_OUTPUT_FILE="$TEST_FILE.test.tokens_output"
+TOKENS_EXPECT_OUTPUT_FILE="$TEST_FILE.test.expected_tokens_output"
+
+TOKENS_OUTPUT_FILE="$TEST_FILE.test.tokens_output"
+TOKENS_EXPECT_OUTPUT_FILE="$TEST_FILE.test.expected_tokens_output"
+
+PROGRAM_OUTPUT="$TEST_FILE.test.program_output"
+
+extract_comment() {
+  tag="$1"
+  comment="# $tag: " 
+  cat $TEST_FILE | grep "$comment" | sed -e "s/$comment//"
+}
+
+# UI
+COLOR_RED=1
+COLOR_GREEN=2
+COLOR_YELLOW=3
+COLOR_BLUE=4
+COLOR_CYAN=6
+COLOR_GRAY=7
+
+colored() {
+    text="$1"
+    color="$2"
+    if [ -t 1 ]; then
+      if tput setaf 1 &> /dev/null; then
+        printf "$(tput setaf $color)$text$(tput sgr0)"
+        return
+      fi
+    fi
+
+    printf "$text"
+}
+
+print_test_description() {
+  colored "$TEST_FILE: \n" $COLOR_CYAN
+  colored "$(extract_comment spec)\n\n" $COLOR_GRAY
+}
+
+print_passed() {
+  context="$1"
+  printf "$context: "
+  colored "passed" $COLOR_GREEN
+  echo
+}
+
+print_failed() {
+  context="$1"
+  printf "$context: "
+  colored "failed" $COLOR_RED
+  echo
+}
+
+print_skiped() {
+  context="$1"
+  printf "$context: "
+  colored "not set" $COLOR_GRAY
+  echo
+}
+
+# Assertions
+expect_exit_code() {
+  context="$1"
+  expected="$2"
+  actual="$3"
+  
+  if [ -z "$expected" ]; then
+    print_skiped "$context"
+    echo "no expected exit code set, it exited with \"$actual\"."
+  else
+    if [ "$expected" = "$actual" ]; then
+      print_passed "$context"
+    else
+      print_failed "$context"
+      colored "Expected exit code: " $COLOR_YELLOW
+      colored "$expected" $COLOR_GRAY
+      colored ", actual: " $COLOR_YELLOW
+      colored "$actual\n" $COLOR_GRAY
+      exit 1
+    fi
+  fi
+}
+
+expect_output() {
+  context="$1"
+  expected_file="$2"
+  actual_file="$3"
+
+  if [ "$(cat $expected_file | wc -l)" = "0" ]; then
+    print_skiped "$context"
+    return
+  fi
+
+  if cmp -s "$expected_file" "$actual_file"; then
+    print_passed "$context"
+  else
+    if [ "$(cat "$expected_file")" = "%empty%" ]; then
+      if [ "$(cat $actual_file | wc -c)" = "0" ]; then
+        print_passed "$context"
+        return
+      fi
+    fi
+    print_failed "$context"
+    colored "$context not match:\n" $COLOR_YELLOW
+    diff "$actual_file" "$expected_file" -u --color
+    exit 1
+  fi
+}
+
+cleanup() {
+  rm -f $PROGRAM*
+}
+
+test_compiler() {
+  $COMPILER_PATH $TEST_FILE -o $PROGRAM > "$COMPILER_OUTPUT_FILE" 2>&1
+
+  EXIT=$?
+  EXPECTED_EXIT=$(extract_comment compiler_exit)
+
+  expect_exit_code compiler_exit "$EXPECTED_EXIT" "$EXIT"
+
+  extract_comment compiler_output > $COMPILER_EXPECT_OUTPUT_FILE
+  expect_output compiler_output "$COMPILER_EXPECT_OUTPUT_FILE" "$COMPILER_OUTPUT_FILE"
+
+  if [ "$EXIT" = "1" ]; then
+    echo "program execution skiped since the compiler failed as expected."
+    cleanup
+    exit 0
+  fi
+}
+
+test_ast() {
+  $COMPILER_PATH $TEST_FILE --dump-ast > "$AST_OUTPUT_FILE" 2>&1
+
+  extract_comment ast > $AST_EXPECT_OUTPUT_FILE
+  expect_output "ast" "$AST_EXPECT_OUTPUT_FILE" "$AST_OUTPUT_FILE"
+}
+
+test_tokens() {
+  extract_comment tokens > $TOKENS_EXPECT_OUTPUT_FILE
+  TOKEN_LINES=$(cat $TOKENS_EXPECT_OUTPUT_FILE | wc -l)
+
+  $COMPILER_PATH $TEST_FILE --dump-tokens | head -n $TOKEN_LINES > "$TOKENS_OUTPUT_FILE" 2>&1
+
+  expect_output tokens $TOKENS_OUTPUT_FILE $TOKENS_EXPECT_OUTPUT_FILE
+
+  print_passed tokens
+}
+
+test_program() {
+  ./$PROGRAM
+
+  EXIT=$?
+  EXPECTED_EXIT=$(extract_comment program_exit)
+  expect_exit_code "program_exit" "$EXPECTED_EXIT" "$EXIT"
+}
+
+
+main() {
+  print_test_description
+
+  test_compiler
+  test_program
+  test_tokens
+  test_ast
+
+  cleanup
+}
+
+main
diff --git a/tests/integration/tests/0001_main_exit.ol b/tests/integration/tests/0001_main_exit.ol
new file mode 100644
index 0000000..c30492f
--- /dev/null
+++ b/tests/integration/tests/0001_main_exit.ol
@@ -0,0 +1,27 @@
+# spec: A minimal olang program
+fn main(): u32 {
+  return 0
+}
+# compiler_exit: 0
+# compiler_output: %empty%
+# program_exit: 0
+# 
+# ast: Translation_Unit
+# ast: `-Function_Definition <name:main> <return:0>
+# ast:   `-Block
+# ast:     `-Return_Statement
+# ast:       `-Literal <kind:u32> <value:0>
+#
+# tokens: ./tests/0001_main_exit.ol:1:32: <line_feed>
+# tokens: ./tests/0001_main_exit.ol:2:1: <fn>
+# tokens: ./tests/0001_main_exit.ol:2:4: <identifier>
+# tokens: ./tests/0001_main_exit.ol:2:8: <(>
+# tokens: ./tests/0001_main_exit.ol:2:9: <)>
+# tokens: ./tests/0001_main_exit.ol:2:10: <:>
+# tokens: ./tests/0001_main_exit.ol:2:12: <identifier>
+# tokens: ./tests/0001_main_exit.ol:2:16: <{>
+# tokens: ./tests/0001_main_exit.ol:2:17: <line_feed>
+# tokens: ./tests/0001_main_exit.ol:3:3: <return>
+# tokens: ./tests/0001_main_exit.ol:3:10: <number>
+# tokens: ./tests/0001_main_exit.ol:3:11: <line_feed>
+# tokens: ./tests/0001_main_exit.ol:4:1: <}>
-- 
2.34.1