Merge branch 'main' of github.com:OrcaLinux/Design-of-Compilers

Author: orcalinux

scanner/build/scanner_test

@@ -1 +1,20 @@
-# Example 1: TINY language code sample
+{Example 1, test all known tokens and 
+multiple spaces and the end}
+x := 5;
+if x<10 then() {simple if 
+condition       }
+    x := 7;
+END {End if}
+
+if (x=7) write x;
+
+read d;
+
+repeat50040;
+repeat 50 until 66
+y = d+9/5
+x=f +y
+            newVar
+
+
+

scanner/examples/example1.txt

@@ -1 +1,6 @@
-# Example 2: TINY language code sample
+{Exampl 2, genral test}
+x := 5; { this is a comment }
+write x;
+read y;
+y : /x; {: is an unknown token} 
+

scanner/examples/example2.txt

@@ -0,0 +1,10 @@
+{Example 3, test IDENTIFIERs with underscore
+and more complex lines}
+abc := 123;
+z := (a + b) * (c / d);
+{ This should be ignored }
+
+{an IDENTIFIER with underscore
+    underscore is an unknown token
+}
+aa_bb = 343 aa55

scanner/examples/example3.txt

@@ -0,0 +1,8 @@
+{Exmaple 4, nested comments and some invalid tokens}
+{ Outer comment { Inner comment } }
+x := 5;
+if x = 10 then y := y + 1;
+x : @$ # ! _ & *^~|||   ;
+
+
+        

scanner/examples/example4.txt

@@ -0,0 +1 @@
+{Example 5, test empty file}

scanner/examples/example5.txt

@@ -1 +1,30 @@
-# Output file for scanner results
+x, IDENTIFIER
+:=, ASSIGN
+5, NUMBER
+;, SEMICOLON
+if, IF
+x, IDENTIFIER
+=, EQUAL
+10, NUMBER
+then, THEN
+y, IDENTIFIER
+:=, ASSIGN
+y, IDENTIFIER
++, PLUS
+1, NUMBER
+;, SEMICOLON
+x, IDENTIFIER
+:, UNKNOWN
+@, UNKNOWN
+$, UNKNOWN
+#, UNKNOWN
+!, UNKNOWN
+_, UNKNOWN
+&, UNKNOWN
+*, MULT
+^, UNKNOWN
+~, UNKNOWN
+|, UNKNOWN
+|, UNKNOWN
+|, UNKNOWN
+;, SEMICOLON

scanner/include/app.hpp

@@ -95,6 +95,14 @@ void App::processTokens(std::string &inputFileContent) {
     tokenStreamBuilder.build();
     std::vector<Token> tokens = tokenStreamBuilder.getTokens();
 
+    // if no tokens are generated, throw an exception
+    if (tokens.empty()) {
+        throw std::runtime_error("No tokens generated. Please check the input file.");
+    }
+
+    // any unknown tokens printed as errors to the console
+    catchUnkonwnTokens(tokens);
+
     // write tokens to output file if specified
     if (hasOutputFile) {
         // set color to orange
@@ -112,6 +120,31 @@ void App::processTokens(std::string &inputFileContent) {
     }
 }
 
+void App::catchUnkonwnTokens(std::vector<TINY::SCANNER::Token> &tokens) {
+    bool hasUnknownTokens = false;
+    for (const Token &token : tokens) {
+        if (token.getType() == TokenType::UNKNOWN) {
+            // for the first unknown token, set color to red
+            if (!hasUnknownTokens) {
+                hasUnknownTokens = true;
+                // set color to red
+                std::cout << "\033[1;31m";
+                std::cerr << "The input contains unexpected tokens, please check the input file."
+                          << std::endl
+                          << "List of unexpected tokens:"
+                          << std::endl;
+            }
+
+            std::cerr << "-\tError: unexpected token '" << token.getValue()
+                      << "' at line " << token.getLine()
+                      << ", column " << token.getColumn()
+                      << std::endl;
+        }
+    }
+    // reset color
+    std::cout << "\033[0m";
+}
+
 void App::printTokens(const std::vector<Token> &tokens, bool includePosition) {
     // set color to Green
     std::cout << "\033[1;32m";