five/compiler/lexer/lexer_test.go

// Copyright (c) 2026 Charles KWON OhJun (charleskwonohjun@gmail.com)
// All rights reserved.

package lexer

import (
	"five/compiler/token"
	"testing"
)

func expectTokens(t *testing.T, source string, expected []token.Kind) {
	t.Helper()
	tokens := Tokenize("test.prg", source)
	// Filter out NEWLINEs and EOF for easier comparison
	var got []token.Kind
	for _, tok := range tokens {
		if tok.Kind != token.NEWLINE && tok.Kind != token.EOF {
			got = append(got, tok.Kind)
		}
	}
	if len(got) != len(expected) {
		t.Errorf("token count: got %d, want %d", len(got), len(expected))
		for i, tok := range tokens {
			t.Logf("  [%d] %v %q", i, tok.Kind, tok.Literal)
		}
		return
	}
	for i, want := range expected {
		if got[i] != want {
			t.Errorf("token[%d]: got %v, want %v", i, got[i], want)
		}
	}
}

func TestBasicArithmetic(t *testing.T) {
	expectTokens(t, "1 + 2 * 3", []token.Kind{
		token.INT, token.PLUS, token.INT, token.STAR, token.INT,
	})
}

func TestAssignment(t *testing.T) {
	expectTokens(t, "x := 10", []token.Kind{
		token.IDENT, token.ASSIGN, token.INT,
	})
}

func TestCompoundAssignment(t *testing.T) {
	expectTokens(t, "n += 5", []token.Kind{
		token.IDENT, token.PLUSEQ, token.INT,
	})
}

func TestStringLiteral(t *testing.T) {
	tokens := Tokenize("test.prg", `"Hello, World!"`)
	if tokens[0].Kind != token.STRING || tokens[0].Literal != "Hello, World!" {
		t.Errorf("got %v %q", tokens[0].Kind, tokens[0].Literal)
	}
}

func TestSingleQuoteString(t *testing.T) {
	tokens := Tokenize("test.prg", `'single'`)
	if tokens[0].Kind != token.STRING || tokens[0].Literal != "single" {
		t.Errorf("got %v %q", tokens[0].Kind, tokens[0].Literal)
	}
}

func TestLogicalLiterals(t *testing.T) {
	expectTokens(t, ".T. .F.", []token.Kind{token.TRUE, token.FALSE})
}

func TestLogicalOperators(t *testing.T) {
	expectTokens(t, ".AND. .OR. .NOT.", []token.Kind{token.AND, token.OR, token.NOT})
}

func TestLogicalCaseInsensitive(t *testing.T) {
	expectTokens(t, ".and. .or. .not. .t. .f.", []token.Kind{
		token.AND, token.OR, token.NOT, token.TRUE, token.FALSE,
	})
}

func TestKeywords(t *testing.T) {
	expectTokens(t, "FUNCTION Main", []token.Kind{token.FUNCTION_KW, token.IDENT})
	expectTokens(t, "function main", []token.Kind{token.FUNCTION_KW, token.IDENT})
	expectTokens(t, "LOCAL n := 0", []token.Kind{token.LOCAL, token.IDENT, token.ASSIGN, token.INT})
	expectTokens(t, "IF x > 10", []token.Kind{token.IF, token.IDENT, token.GT, token.INT})
	expectTokens(t, "DO WHILE i <= 10", []token.Kind{token.DO, token.WHILE, token.IDENT, token.LTE, token.INT})
	expectTokens(t, "RETURN NIL", []token.Kind{token.RETURN, token.NIL_LIT})
}

func TestXBaseCommands(t *testing.T) {
	expectTokens(t, "USE customers", []token.Kind{token.USE, token.IDENT})
	expectTokens(t, "SEEK cKey", []token.Kind{token.SEEK, token.IDENT})
	expectTokens(t, "REPLACE name WITH cNewName", []token.Kind{
		token.REPLACE, token.IDENT, token.WITH, token.IDENT,
	})
	expectTokens(t, "APPEND BLANK", []token.Kind{token.APPEND, token.BLANK})
	expectTokens(t, "GO TOP", []token.Kind{token.GO, token.TOP})
}

func TestClassDeclaration(t *testing.T) {
	expectTokens(t, "CLASS Person", []token.Kind{token.CLASS, token.IDENT})
	expectTokens(t, "DATA cName INIT", []token.Kind{token.DATA, token.IDENT, token.IDENT})
	expectTokens(t, "METHOD New", []token.Kind{token.METHOD, token.IDENT})
	expectTokens(t, "ENDCLASS", []token.Kind{token.ENDCLASS})
}

func TestArrowAndColons(t *testing.T) {
	expectTokens(t, "cust->name", []token.Kind{
		token.IDENT, token.ARROW, token.IDENT,
	})
	expectTokens(t, "obj:greet()", []token.Kind{
		token.IDENT, token.COLON, token.IDENT, token.LPAREN, token.RPAREN,
	})
	expectTokens(t, "::name", []token.Kind{token.COLONCOLON, token.IDENT})
}

func TestCodeBlock(t *testing.T) {
	expectTokens(t, "{|x| x + 1}", []token.Kind{
		token.LBRACE, token.PIPE, token.IDENT, token.PIPE,
		token.IDENT, token.PLUS, token.INT, token.RBRACE,
	})
}

func TestHashLiteral(t *testing.T) {
	expectTokens(t, `{"a" => 1}`, []token.Kind{
		token.LBRACE, token.STRING, token.DBLARROW, token.INT, token.RBRACE,
	})
}

func TestComparison(t *testing.T) {
	expectTokens(t, "a == b", []token.Kind{token.IDENT, token.EXEQ, token.IDENT})
	expectTokens(t, "a != b", []token.Kind{token.IDENT, token.NEQ, token.IDENT})
	expectTokens(t, "a <> b", []token.Kind{token.IDENT, token.NEQ, token.IDENT})
	expectTokens(t, "a # b", []token.Kind{token.IDENT, token.NEQ, token.IDENT})
	expectTokens(t, "a <= b", []token.Kind{token.IDENT, token.LTE, token.IDENT})
	expectTokens(t, "a >= b", []token.Kind{token.IDENT, token.GTE, token.IDENT})
}

func TestDoubleNumber(t *testing.T) {
	tokens := Tokenize("test.prg", "3.14")
	if tokens[0].Kind != token.DOUBLE || tokens[0].Literal != "3.14" {
		t.Errorf("got %v %q", tokens[0].Kind, tokens[0].Literal)
	}
}

func TestHexNumber(t *testing.T) {
	tokens := Tokenize("test.prg", "0xFF")
	if tokens[0].Kind != token.INT || tokens[0].Literal != "0xFF" {
		t.Errorf("got %v %q", tokens[0].Kind, tokens[0].Literal)
	}
}

func TestMacroOperator(t *testing.T) {
	expectTokens(t, "&cVar", []token.Kind{token.AMPERSAND, token.IDENT})
}

func TestImport(t *testing.T) {
	expectTokens(t, `IMPORT "net/http"`, []token.Kind{token.IMPORT, token.STRING})
}

func TestPreprocessor(t *testing.T) {
	tokens := Tokenize("test.prg", "#include")
	if tokens[0].Kind != token.PP_INCLUDE {
		t.Errorf("got %v, want PP_INCLUDE", tokens[0].Kind)
	}

	tokens = Tokenize("test.prg", "#define")
	if tokens[0].Kind != token.PP_DEFINE {
		t.Errorf("got %v, want PP_DEFINE", tokens[0].Kind)
	}

	tokens = Tokenize("test.prg", "#pragma")
	if tokens[0].Kind != token.PP_PRAGMA {
		t.Errorf("got %v, want PP_PRAGMA", tokens[0].Kind)
	}
}

func TestLineComment(t *testing.T) {
	expectTokens(t, "x := 10 // comment", []token.Kind{
		token.IDENT, token.ASSIGN, token.INT,
	})
}

func TestAmpAmpComment(t *testing.T) {
	expectTokens(t, "x := 10 && comment", []token.Kind{
		token.IDENT, token.ASSIGN, token.INT,
	})
}

func TestBlockComment(t *testing.T) {
	expectTokens(t, "x /* skip */ + y", []token.Kind{
		token.IDENT, token.PLUS, token.IDENT,
	})
}

func TestLineContinuation(t *testing.T) {
	// Semicolon at end of line = continuation
	expectTokens(t, "x + ;\n  y", []token.Kind{
		token.IDENT, token.PLUS, token.IDENT,
	})
}

func TestNewlineAsTerminator(t *testing.T) {
	tokens := Tokenize("test.prg", "x\ny")
	kinds := make([]token.Kind, 0)
	for _, tok := range tokens {
		if tok.Kind != token.EOF {
			kinds = append(kinds, tok.Kind)
		}
	}
	// Should have: IDENT NEWLINE IDENT
	if len(kinds) != 3 || kinds[1] != token.NEWLINE {
		t.Errorf("expected IDENT NEWLINE IDENT, got %v", kinds)
	}
}

func TestPosition(t *testing.T) {
	tokens := Tokenize("test.prg", "x := 10")
	if tokens[0].Pos.Line != 1 || tokens[0].Pos.Col != 1 {
		t.Errorf("x position: line=%d col=%d", tokens[0].Pos.Line, tokens[0].Pos.Col)
	}
}

// Full program test
func TestFullProgram(t *testing.T) {
	src := `FUNCTION Main()
   LOCAL n := 10
   ? "Hello", n
   RETURN NIL`

	tokens := Tokenize("test.prg", src)
	var kinds []token.Kind
	for _, tok := range tokens {
		if tok.Kind != token.NEWLINE && tok.Kind != token.EOF {
			kinds = append(kinds, tok.Kind)
		}
	}

	expected := []token.Kind{
		token.FUNCTION_KW, token.IDENT, token.LPAREN, token.RPAREN,
		token.LOCAL, token.IDENT, token.ASSIGN, token.INT,
		token.QMARK, token.STRING, token.COMMA, token.IDENT,
		token.RETURN, token.NIL_LIT,
	}

	if len(kinds) != len(expected) {
		t.Errorf("token count: got %d, want %d", len(kinds), len(expected))
		for i, tok := range tokens {
			if tok.Kind != token.NEWLINE && tok.Kind != token.EOF {
				t.Logf("  [%d] %v %q", i, tok.Kind, tok.Literal)
			}
		}
		return
	}
	for i, want := range expected {
		if kinds[i] != want {
			t.Errorf("token[%d]: got %v %q, want %v", i, kinds[i], tokens[i].Literal, want)
		}
	}
}