fivedev/five
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Phase 3 — #25,28,29,41 token/AST/parser cleanup

Browse Source 
Files modified (4):
  compiler/token/token.go — #25: Replace hand-rolled itoa with strconv.Itoa
    Fixes math.MinInt overflow bug in original implementation
  compiler/ast/ast.go — #29: Fix VarDecl.End() returning last var position
    Was returning Pos() (useless span info)
  compiler/parser/stmtreg.go — #28: Eliminate all 7 token array mutations
    rewriteAsIdent() modifies p.current only, not the token array
    Prevents backtracking corruption and improves safety
  compiler/lexer/lexer.go — Already clean from Phase 2

Issues resolved: #25 (MEDIUM), #28 (MEDIUM), #29 (MEDIUM), #41 partial (LOW)
Total fixed: 29/53

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Charles KWON OhJun
2026-04-01 11:58:20 +09:00
parent f950cb0784
commit 6ffcf77dd8
4 changed files with 103 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
compiler/ast/ast.go
View File

@@ -117,7 +117,12 @@ type VarDecl struct {
}
func (d *VarDecl) Pos() token.Position { return d.DeclPos }
func (d *VarDecl) End() token.Position { return d.DeclPos }
func (d *VarDecl) End() token.Position {
if len(d.Vars) > 0 {
return d.Vars[len(d.Vars)-1].NamePos
}
return d.DeclPos
}
func (d *VarDecl) declNode() {}
func (d *VarDecl) stmtNode() {} // PRIVATE/PUBLIC can appear as statements

34
compiler/parser/stmtreg.go
View File

@@ -88,6 +88,13 @@ func (p *Parser) lookupStmtParser() StmtParser {
// --- Thin wrappers: each calls the existing parse method ---
// rewriteAsIdent treats the current keyword token as an identifier for expression parsing.
// Instead of mutating the token array (which breaks backtracking), it temporarily
// overwrites the current token and restores nothing — the token is consumed immediately.
func (p *Parser) rewriteAsIdent(name string) {
p.current = token.Token{Kind: token.IDENT, Literal: name, Pos: p.current.Pos}
}
func (p *Parser) stmtIf() ast.Stmt {
if p.peekAt(1) == token.LPAREN {
if p.looksLikeIIF() {
@@ -99,9 +106,7 @@ func (p *Parser) stmtIf() ast.Stmt {
func (p *Parser) stmtDo() ast.Stmt {
if p.peekAt(1) == token.LPAREN {
p.tokens[p.pos].Kind = token.IDENT
p.tokens[p.pos].Literal = "Do"
p.current = p.tokens[p.pos]
p.rewriteAsIdent("Do")
return p.parseExprStmt()
}
if p.peekAt(1) == token.CASE || token.LookupKeyword(p.peekLitAt(1)) == token.CASE {
@@ -118,9 +123,7 @@ func (p *Parser) stmtDo() ast.Stmt {
func (p *Parser) stmtWhile() ast.Stmt {
if p.peekAt(1) == token.LPAREN {
p.tokens[p.pos].Kind = token.IDENT
p.tokens[p.pos].Literal = "While"
p.current = p.tokens[p.pos]
p.rewriteAsIdent("While")
return p.parseExprStmt()
}
return p.parseDoWhile()
@@ -130,9 +133,7 @@ func (p *Parser) stmtFor() ast.Stmt {
next := p.peekAt(1)
if next == token.ASSIGN || next == token.LPAREN ||
next == token.PLUSEQ || next == token.MINUSEQ {
p.tokens[p.pos].Kind = token.IDENT
p.tokens[p.pos].Literal = "for"
p.current = p.tokens[p.pos]
p.rewriteAsIdent("for")
return p.parseExprStmt()
}
return p.parseFor()
@@ -140,9 +141,7 @@ func (p *Parser) stmtFor() ast.Stmt {
func (p *Parser) stmtBegin() ast.Stmt {
if p.peekAt(1) != token.SEQUENCE && p.peekAt(1) != token.NEWLINE && p.peekAt(1) != token.EOF {
p.tokens[p.pos].Kind = token.IDENT
p.tokens[p.pos].Literal = "begin"
p.current = p.tokens[p.pos]
p.rewriteAsIdent("begin")
return p.parseExprStmt()
}
return p.parseBeginSequence()
@@ -153,9 +152,7 @@ func (p *Parser) stmtSwitch() ast.Stmt { return p.parseSwitch() }
func (p *Parser) stmtReturn() ast.Stmt {
next := p.peekAt(1)
if next == token.ASSIGN || next == token.PLUSEQ || next == token.MINUSEQ {
p.tokens[p.pos].Kind = token.IDENT
p.tokens[p.pos].Literal = "return"
p.current = p.tokens[p.pos]
p.rewriteAsIdent("return")
return p.parseExprStmt()
}
return p.parseReturn()
@@ -179,8 +176,7 @@ func (p *Parser) stmtPublic() ast.Stmt { return p.parsePrivatePublic(ast.ScopeP
func (p *Parser) stmtVarDecl() ast.Stmt { return p.parseVarDecl() }
func (p *Parser) stmtParameters() ast.Stmt {
p.tokens[p.pos].Kind = token.LOCAL
p.current = p.tokens[p.pos]
p.current = token.Token{Kind: token.LOCAL, Literal: "LOCAL", Pos: p.current.Pos}
return p.parseVarDecl()
}
@@ -201,9 +197,7 @@ func (p *Parser) stmtAt() ast.Stmt { return p.parseAtCmd() }
func (p *Parser) stmtGo() ast.Stmt {
if p.current.Kind == token.GO && p.peekAt(1) == token.LPAREN {
p.tokens[p.pos].Kind = token.IDENT
p.tokens[p.pos].Literal = "Go"
p.current = p.tokens[p.pos]
p.rewriteAsIdent("Go")
return p.parseExprStmt()
}
return p.parseGo()

29
compiler/token/token.go
View File

@@ -6,6 +6,8 @@
// (ref/typescript-go/internal/ast/kind.go, precedence.go).
package token
import "strconv"
// Kind represents a token type. Using int16 following tsgo pattern.
type Kind int16
@@ -215,33 +217,12 @@ type Position struct {
func (p Position) String() string {
if p.File != "" {
return p.File + ":" + itoa(p.Line) + ":" + itoa(p.Col)
return p.File + ":" + strconv.Itoa(p.Line) + ":" + strconv.Itoa(p.Col)
}
return itoa(p.Line) + ":" + itoa(p.Col)
return strconv.Itoa(p.Line) + ":" + strconv.Itoa(p.Col)
}
// simple int-to-string without importing strconv
func itoa(n int) string {
if n == 0 {
return "0"
}
buf := [20]byte{}
i := len(buf) - 1
neg := n < 0
if neg {
n = -n
}
for n > 0 {
buf[i] = byte('0' + n%10)
i--
n /= 10
}
if neg {
buf[i] = '-'
i--
}
return string(buf[i+1:])
}
// itoa removed — using strconv.Itoa (fixes math.MinInt overflow bug)
// --- Operator Precedence (tsgo pattern) ---

84
docs/.pdca-status.json
View File

@@ -1,6 +1,6 @@
{
"version": "2.0",
"lastUpdated": "2026-04-01T02:46:45.142Z",
"lastUpdated": "2026-04-01T02:57:07.253Z",
"activeFeatures": [
"hbrt",
"hbrtl",
@@ -72,7 +72,7 @@
"documents": {},
"timestamps": {
"started": "2026-03-27T11:22:43.083Z",
"lastUpdated": "2026-03-30T15:57:05.054Z"
"lastUpdated": "2026-04-01T02:53:26.889Z"
},
"lastFile": "/mnt/d/charles/five/compiler/token/token.go"
},
@@ -98,7 +98,7 @@
"documents": {},
"timestamps": {
"started": "2026-03-27T11:34:07.187Z",
"lastUpdated": "2026-03-30T15:47:26.151Z"
"lastUpdated": "2026-04-01T02:53:52.639Z"
},
"lastFile": "/mnt/d/charles/five/compiler/ast/ast.go"
},
@@ -111,9 +111,9 @@
"documents": {},
"timestamps": {
"started": "2026-03-27T11:38:35.393Z",
"lastUpdated": "2026-04-01T01:28:45.794Z"
"lastUpdated": "2026-04-01T02:57:07.253Z"
},
"lastFile": "/mnt/d/charles/five/compiler/parser/expr.go"
"lastFile": "/mnt/d/charles/five/compiler/parser/stmtreg.go"
},
"gengo": {
"phase": "do",
@@ -280,7 +280,7 @@
"session": {
"startedAt": "2026-03-27T06:06:49.620Z",
"onboardingCompleted": false,
"lastActivity": "2026-04-01T02:46:45.142Z"
"lastActivity": "2026-04-01T02:57:07.253Z"
},
"history": [
{
@@ -5694,6 +5694,78 @@
"feature": "gengo",
"phase": "do",
"action": "updated"
},
{
"timestamp": "2026-04-01T02:52:55.069Z",
"feature": "token",
"phase": "do",
"action": "updated"
},
{
"timestamp": "2026-04-01T02:53:11.673Z",
"feature": "token",
"phase": "do",
"action": "updated"
},
{
"timestamp": "2026-04-01T02:53:26.889Z",
"feature": "token",
"phase": "do",
"action": "updated"
},
{
"timestamp": "2026-04-01T02:53:52.639Z",
"feature": "ast",
"phase": "do",
"action": "updated"
},
{
"timestamp": "2026-04-01T02:54:30.018Z",
"feature": "parser",
"phase": "do",
"action": "updated"
},
{
"timestamp": "2026-04-01T02:55:12.388Z",
"feature": "parser",
"phase": "do",
"action": "updated"
},
{
"timestamp": "2026-04-01T02:55:24.710Z",
"feature": "parser",
"phase": "do",
"action": "updated"
},
{
"timestamp": "2026-04-01T02:55:38.650Z",
"feature": "parser",
"phase": "do",
"action": "updated"
},
{
"timestamp": "2026-04-01T02:55:52.312Z",
"feature": "parser",
"phase": "do",
"action": "updated"
},
{
"timestamp": "2026-04-01T02:56:06.938Z",
"feature": "parser",
"phase": "do",
"action": "updated"
},
{
"timestamp": "2026-04-01T02:56:42.883Z",
"feature": "parser",
"phase": "do",
"action": "updated"
},
{
"timestamp": "2026-04-01T02:57:07.253Z",
"feature": "parser",
"phase": "do",
"action": "updated"
}
]
}