// Copyright (c) 2026 Charles KWON OhJun (charleskwonohjun@gmail.com) // All rights reserved. package lexer import ( "five/compiler/token" "testing" ) func expectTokens(t *testing.T, source string, expected []token.Kind) { t.Helper() tokens := Tokenize("test.prg", source) // Filter out NEWLINEs and EOF for easier comparison var got []token.Kind for _, tok := range tokens { if tok.Kind != token.NEWLINE && tok.Kind != token.EOF { got = append(got, tok.Kind) } } if len(got) != len(expected) { t.Errorf("token count: got %d, want %d", len(got), len(expected)) for i, tok := range tokens { t.Logf(" [%d] %v %q", i, tok.Kind, tok.Literal) } return } for i, want := range expected { if got[i] != want { t.Errorf("token[%d]: got %v, want %v", i, got[i], want) } } } func TestBasicArithmetic(t *testing.T) { expectTokens(t, "1 + 2 * 3", []token.Kind{ token.INT, token.PLUS, token.INT, token.STAR, token.INT, }) } func TestAssignment(t *testing.T) { expectTokens(t, "x := 10", []token.Kind{ token.IDENT, token.ASSIGN, token.INT, }) } func TestCompoundAssignment(t *testing.T) { expectTokens(t, "n += 5", []token.Kind{ token.IDENT, token.PLUSEQ, token.INT, }) } func TestStringLiteral(t *testing.T) { tokens := Tokenize("test.prg", `"Hello, World!"`) if tokens[0].Kind != token.STRING || tokens[0].Literal != "Hello, World!" { t.Errorf("got %v %q", tokens[0].Kind, tokens[0].Literal) } } func TestSingleQuoteString(t *testing.T) { tokens := Tokenize("test.prg", `'single'`) if tokens[0].Kind != token.STRING || tokens[0].Literal != "single" { t.Errorf("got %v %q", tokens[0].Kind, tokens[0].Literal) } } func TestLogicalLiterals(t *testing.T) { expectTokens(t, ".T. .F.", []token.Kind{token.TRUE, token.FALSE}) } func TestLogicalOperators(t *testing.T) { expectTokens(t, ".AND. .OR. .NOT.", []token.Kind{token.AND, token.OR, token.NOT}) } func TestLogicalCaseInsensitive(t *testing.T) { expectTokens(t, ".and. .or. .not. .t. .f.", []token.Kind{ token.AND, token.OR, token.NOT, token.TRUE, token.FALSE, }) } func TestKeywords(t *testing.T) { expectTokens(t, "FUNCTION Main", []token.Kind{token.FUNCTION_KW, token.IDENT}) expectTokens(t, "function main", []token.Kind{token.FUNCTION_KW, token.IDENT}) expectTokens(t, "LOCAL n := 0", []token.Kind{token.LOCAL, token.IDENT, token.ASSIGN, token.INT}) expectTokens(t, "IF x > 10", []token.Kind{token.IF, token.IDENT, token.GT, token.INT}) expectTokens(t, "DO WHILE i <= 10", []token.Kind{token.DO, token.WHILE, token.IDENT, token.LTE, token.INT}) expectTokens(t, "RETURN NIL", []token.Kind{token.RETURN, token.NIL_LIT}) } func TestXBaseCommands(t *testing.T) { expectTokens(t, "USE customers", []token.Kind{token.USE, token.IDENT}) expectTokens(t, "SEEK cKey", []token.Kind{token.SEEK, token.IDENT}) expectTokens(t, "REPLACE name WITH cNewName", []token.Kind{ token.REPLACE, token.IDENT, token.WITH, token.IDENT, }) expectTokens(t, "APPEND BLANK", []token.Kind{token.APPEND, token.BLANK}) expectTokens(t, "GO TOP", []token.Kind{token.GO, token.TOP}) } func TestClassDeclaration(t *testing.T) { expectTokens(t, "CLASS Person", []token.Kind{token.CLASS, token.IDENT}) expectTokens(t, "DATA cName INIT", []token.Kind{token.DATA, token.IDENT, token.IDENT}) expectTokens(t, "METHOD New", []token.Kind{token.METHOD, token.IDENT}) expectTokens(t, "ENDCLASS", []token.Kind{token.ENDCLASS}) } func TestArrowAndColons(t *testing.T) { expectTokens(t, "cust->name", []token.Kind{ token.IDENT, token.ARROW, token.IDENT, }) expectTokens(t, "obj:greet()", []token.Kind{ token.IDENT, token.COLON, token.IDENT, token.LPAREN, token.RPAREN, }) expectTokens(t, "::name", []token.Kind{token.COLONCOLON, token.IDENT}) } func TestCodeBlock(t *testing.T) { expectTokens(t, "{|x| x + 1}", []token.Kind{ token.LBRACE, token.PIPE, token.IDENT, token.PIPE, token.IDENT, token.PLUS, token.INT, token.RBRACE, }) } func TestHashLiteral(t *testing.T) { expectTokens(t, `{"a" => 1}`, []token.Kind{ token.LBRACE, token.STRING, token.DBLARROW, token.INT, token.RBRACE, }) } func TestComparison(t *testing.T) { expectTokens(t, "a == b", []token.Kind{token.IDENT, token.EXEQ, token.IDENT}) expectTokens(t, "a != b", []token.Kind{token.IDENT, token.NEQ, token.IDENT}) expectTokens(t, "a <> b", []token.Kind{token.IDENT, token.NEQ, token.IDENT}) expectTokens(t, "a # b", []token.Kind{token.IDENT, token.NEQ, token.IDENT}) expectTokens(t, "a <= b", []token.Kind{token.IDENT, token.LTE, token.IDENT}) expectTokens(t, "a >= b", []token.Kind{token.IDENT, token.GTE, token.IDENT}) } func TestDoubleNumber(t *testing.T) { tokens := Tokenize("test.prg", "3.14") if tokens[0].Kind != token.DOUBLE || tokens[0].Literal != "3.14" { t.Errorf("got %v %q", tokens[0].Kind, tokens[0].Literal) } } func TestHexNumber(t *testing.T) { tokens := Tokenize("test.prg", "0xFF") if tokens[0].Kind != token.INT || tokens[0].Literal != "0xFF" { t.Errorf("got %v %q", tokens[0].Kind, tokens[0].Literal) } } func TestMacroOperator(t *testing.T) { expectTokens(t, "&cVar", []token.Kind{token.AMPERSAND, token.IDENT}) } func TestImport(t *testing.T) { expectTokens(t, `IMPORT "net/http"`, []token.Kind{token.IMPORT, token.STRING}) } func TestPreprocessor(t *testing.T) { tokens := Tokenize("test.prg", "#include") if tokens[0].Kind != token.PP_INCLUDE { t.Errorf("got %v, want PP_INCLUDE", tokens[0].Kind) } tokens = Tokenize("test.prg", "#define") if tokens[0].Kind != token.PP_DEFINE { t.Errorf("got %v, want PP_DEFINE", tokens[0].Kind) } tokens = Tokenize("test.prg", "#pragma") if tokens[0].Kind != token.PP_PRAGMA { t.Errorf("got %v, want PP_PRAGMA", tokens[0].Kind) } } func TestLineComment(t *testing.T) { expectTokens(t, "x := 10 // comment", []token.Kind{ token.IDENT, token.ASSIGN, token.INT, }) } func TestAmpAmpComment(t *testing.T) { expectTokens(t, "x := 10 && comment", []token.Kind{ token.IDENT, token.ASSIGN, token.INT, }) } func TestBlockComment(t *testing.T) { expectTokens(t, "x /* skip */ + y", []token.Kind{ token.IDENT, token.PLUS, token.IDENT, }) } func TestLineContinuation(t *testing.T) { // Semicolon at end of line = continuation expectTokens(t, "x + ;\n y", []token.Kind{ token.IDENT, token.PLUS, token.IDENT, }) } func TestNewlineAsTerminator(t *testing.T) { tokens := Tokenize("test.prg", "x\ny") kinds := make([]token.Kind, 0) for _, tok := range tokens { if tok.Kind != token.EOF { kinds = append(kinds, tok.Kind) } } // Should have: IDENT NEWLINE IDENT if len(kinds) != 3 || kinds[1] != token.NEWLINE { t.Errorf("expected IDENT NEWLINE IDENT, got %v", kinds) } } func TestPosition(t *testing.T) { tokens := Tokenize("test.prg", "x := 10") if tokens[0].Pos.Line != 1 || tokens[0].Pos.Col != 1 { t.Errorf("x position: line=%d col=%d", tokens[0].Pos.Line, tokens[0].Pos.Col) } } // Full program test func TestFullProgram(t *testing.T) { src := `FUNCTION Main() LOCAL n := 10 ? "Hello", n RETURN NIL` tokens := Tokenize("test.prg", src) var kinds []token.Kind for _, tok := range tokens { if tok.Kind != token.NEWLINE && tok.Kind != token.EOF { kinds = append(kinds, tok.Kind) } } expected := []token.Kind{ token.FUNCTION_KW, token.IDENT, token.LPAREN, token.RPAREN, token.LOCAL, token.IDENT, token.ASSIGN, token.INT, token.QMARK, token.STRING, token.COMMA, token.IDENT, token.RETURN, token.NIL_LIT, } if len(kinds) != len(expected) { t.Errorf("token count: got %d, want %d", len(kinds), len(expected)) for i, tok := range tokens { if tok.Kind != token.NEWLINE && tok.Kind != token.EOF { t.Logf(" [%d] %v %q", i, tok.Kind, tok.Literal) } } return } for i, want := range expected { if kinds[i] != want { t.Errorf("token[%d]: got %v %q, want %v", i, kinds[i], tokens[i].Literal, want) } } }