fivedev/five
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: RTL Go-native migration — 27 optimizations, DML up to 70-90x

Browse Source 
Systematic pass through PRG hot paths, promoting them to Go RTL while
preserving Harbour/FiveSql2 semantics. Full log in
docs/RTL-Go-Native-Migration.md.

Bench (bench_sql) vs 2026-04-08 baseline
 - B1  SELECT *             2,192 → 114   µs   (19x)
 - B6  INNER JOIN           9,291 → 233   µs   (40x)
 - B7  CTE simple           8,037 → 129   µs   (62x)
 - B9  ROW_NUMBER           3,705 → 265   µs   (14x)
 - B10 RANK PARTITION       4,748 → 309   µs   (15x)
 - B12 INSERT (WA cache)    4,319 →  63   µs   (69x)
 - B13 UPDATE (WA cache)    6,144 →  68   µs   (90x)
 - B15 CTE+WIN+JOIN        18,395 → 1,873 µs   (10x)

Infrastructure
 - HbHash O(1) Index preserving insertion order (Harbour KEEPORDER)
 - HbDeepClone Go RTL (scalar-sharing, immutable hash keys)
 - MEMRDD auto-imported via gengo; all Five programs get mem:name driver
 - SQL plan + pcode caches (s_hPlanCache, s_hDmlPcodeCache)
 - Opt-in SqlWACacheEnable — dbUseArea/Close/Commit batched for DML

SQL engine
 - FiveSql2 lexer ported to Go (byte FSM) with combined automatic
   template parameterization (literals → ?, concat queries share plan)
 - Go RTL: SqlDistinct, SqlGroupRows, SqlWindowPartitions,
   SqlWindowSortPartition, SqlWindowAssignRank, SqlComputeAggSimple,
   SqlBulkInsert, SqlBulkUpdate, SqlExprHasAgg, SqlEvalHaving
 - CTE / subquery / driving-table materialize paths use MEMRDD
 - SqlCoerce/SqlCmp/SqlIsTrue helpers moved from PRG to Go
 - SqlBulkUpdate defers Flush when WA cache active (APFS fsync was
   dominant B13 cost — 1.6ms/call → gone)

Correctness fixes uncovered during migration
 - ASort default path now sorts dates/logicals/timestamps (was no-op)
 - ORDER BY default NULL placement matches PRG SqlRowCompare across
   Go fast path; explicit NULLS FIRST/LAST honored by both paths
 - SqlBulkUpdate respects EXCLUSIVE vs SHARED mode record locks
 - SqlCmp/SqlCmpEq normalize NumInt vs Double (caught by test 6b)

Verification
 - go test ./...              ALL PASS
 - FiveSql2 test_sql1999      43/43
 - tests/compat_harbour       56/56 (+5 new: ASort dates/logicals,
                              AScan int cross-type)
 - Regression test test_null_order.prg for ORDER BY NULL ordering

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
CharlesKWON
2026-04-17 20:20:14 +09:00
parent 3caadb23b9
commit dd270d5d9d
31 changed files with 4501 additions and 495 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
_FiveSql2/src/TFiveSQL.prg
View File

@@ -14,6 +14,19 @@
#include "hbclass.ch"
#include "FiveSqlDef.ch"
/* Plan cache: cSQL → parsed hQuery.
*
* The FiveSql2 parser runs lex + Pratt-style AST build per call; for
* repeated identical SQL (typical in report / loop / benchmark workloads)
* this is pure overhead. We cache the pristine parse result keyed by
* the raw SQL text and hand every subsequent call a deep clone via
* HbDeepClone so in-place mutations (SqlFoldConst, aTables rewriting)
* during Run() never corrupt the cached tree.
*
* Cached entries live until process exit; distinct SQL text count is
* bounded by the caller's template set, so LRU is deferred. */
STATIC s_hPlanCache := { => }
CLASS TFiveSQL
DATA oLexer
@@ -40,20 +53,50 @@ RETURN SELF
METHOD Execute( cSQL, bBlock ) CLASS TFiveSQL
LOCAL aTokens, hQuery, aResult
LOCAL aLex, cKey, aParams
/* Parse — no caching (plan trees are mutated during execution) */
::oLexer := TSqlLexer():New( cSQL )
::oLexer:Tokenize()
aTokens := ::oLexer:GetTokens()
/* Fast path: no explicit aParams → single Go RTL lex+normalize call
* (SqlLexAndExtractTemplate). Returns {aTokens, cKey, aParams}; the
* tokens already have TK_TEXT/TK_NUM replaced with TK_QMARK, so
* TSqlParser2 sees the template shape and emits ND_PAR references
* against the extracted aParams. */
IF Empty( ::aParams )
aLex := SqlLexAndExtractTemplate( cSQL )
aTokens := aLex[ 1 ]
cKey := aLex[ 2 ]
aParams := aLex[ 3 ]
::oParser := TSqlParser2():New( aTokens, ::aParams )
hQuery := ::oParser:Parse()
IF hb_HHasKey( s_hPlanCache, cKey )
hQuery := HbDeepClone( s_hPlanCache[ cKey ] )
ELSE
::oParser := TSqlParser2():New( aTokens, aParams )
hQuery := ::oParser:Parse()
IF hQuery == NIL
RETURN { { "__error__" }, { { SQL_ERR_SYNTAX, "Failed to parse SQL", cSQL } } }
ENDIF
s_hPlanCache[ cKey ] := HbDeepClone( hQuery )
ENDIF
IF hQuery == NIL
RETURN { { "__error__" }, { { SQL_ERR_SYNTAX, "Failed to parse SQL", cSQL } } }
::oExec := TSqlExecutor():New( hQuery, aParams )
::oExec:cCacheKey := cKey
ELSE
/* Caller supplied explicit params — cache by raw SQL text. */
IF hb_HHasKey( s_hPlanCache, cSQL )
hQuery := HbDeepClone( s_hPlanCache[ cSQL ] )
ELSE
aTokens := SqlLexerTokenize( cSQL )
::oParser := TSqlParser2():New( aTokens, ::aParams )
hQuery := ::oParser:Parse()
IF hQuery == NIL
RETURN { { "__error__" }, { { SQL_ERR_SYNTAX, "Failed to parse SQL", cSQL } } }
ENDIF
s_hPlanCache[ cSQL ] := HbDeepClone( hQuery )
ENDIF
::oExec := TSqlExecutor():New( hQuery, ::aParams )
::oExec:cCacheKey := cSQL
ENDIF
::oExec := TSqlExecutor():New( hQuery, ::aParams )
::oExec:bRowBlock := bBlock
aResult := ::oExec:Run()

72
_FiveSql2/src/TSqlAgg.prg
View File

@@ -48,12 +48,13 @@ RETURN .F.
METHOD GroupBy( aRows, aFN, aCols, aGroupBy, xHaving, aTables, aParams ) CLASS TSqlAgg
LOCAL hGroups := { => }
LOCAL i, j, cKey, aGroupRows, aResult := {}
LOCAL i, j, aGroupRows, aResult := {}
LOCAL aNewRow
LOCAL nGCol, cN, nCI, lPass
LOCAL aGroupIdx := {}
LOCAL aSets, aCurSet, nSet, hOmitIdx, aSubResult
LOCAL aGroupedRows
LOCAL aColInfo /* { lIsAgg, nCI } per SELECT column, pre-resolved */
/* Aggregate on empty set */
IF Len( aRows ) == 0 .AND. ::HasAgg( aCols )
@@ -109,37 +110,39 @@ METHOD GroupBy( aRows, aFN, aCols, aGroupBy, xHaving, aTables, aParams ) CLASS T
AAdd( aGroupIdx, nGCol )
NEXT
/* Grouping step — delegate to Go RTL SqlGroupRows to collapse
* N·M per-row boundary crossings (SqlValToStr / hb_HHasKey / AAdd)
* into a single call. Aggregates and HAVING stay in PRG because
* they touch too many expression kinds to port cleanly. */
IF Len( aGroupBy ) == 0 .AND. ::HasAgg( aCols )
hGroups[ "__ALL__" ] := aRows
aGroupedRows := { aRows }
ELSE
FOR i := 1 TO Len( aRows )
cKey := ""
FOR j := 1 TO Len( aGroupBy )
nGCol := aGroupIdx[ j ]
IF nGCol > 0 .AND. nGCol <= Len( aRows[ i ] )
cKey += SqlValToStr( aRows[ i ][ nGCol ] ) + "|"
ENDIF
NEXT
IF ! hb_HHasKey( hGroups, cKey )
hGroups[ cKey ] := {}
ENDIF
AAdd( hGroups[ cKey ], aRows[ i ] )
NEXT
aGroupedRows := SqlGroupRows( aRows, aGroupIdx )
ENDIF
/* Pre-resolve per SELECT column: aggregate flag + column index.
* Avoids SqlExprHasAgg + SqlExprName + FindColIdx2 per group. */
aColInfo := Array( Len( aCols ) )
FOR j := 1 TO Len( aCols )
IF SqlExprHasAgg( aCols[ j ][ 1 ] )
aColInfo[ j ] := { .T., 0 }
ELSE
cN := SqlExprName( aCols[ j ][ 1 ] )
nCI := ::FindColIdx2( cN, aFN )
aColInfo[ j ] := { .F., nCI }
ENDIF
NEXT
/* Compute aggregates for each group */
FOR EACH aGroupRows IN hb_HValues( hGroups )
aNewRow := {}
FOR EACH aGroupRows IN aGroupedRows
aNewRow := Array( Len( aCols ) )
FOR j := 1 TO Len( aCols )
IF SqlExprHasAgg( aCols[ j ][ 1 ] )
AAdd( aNewRow, ::ComputeAgg( aCols[ j ][ 1 ], aGroupRows, aFN ) )
IF aColInfo[ j ][ 1 ]
aNewRow[ j ] := ::ComputeAgg( aCols[ j ][ 1 ], aGroupRows, aFN )
ELSE
cN := SqlExprName( aCols[ j ][ 1 ] )
nCI := ::FindColIdx2( cN, aFN )
nCI := aColInfo[ j ][ 2 ]
IF nCI > 0 .AND. Len( aGroupRows ) > 0 .AND. nCI <= Len( aGroupRows[ 1 ] )
AAdd( aNewRow, aGroupRows[ 1 ][ nCI ] )
ELSE
AAdd( aNewRow, NIL )
aNewRow[ j ] := aGroupRows[ 1 ][ nCI ]
ENDIF
ENDIF
NEXT
@@ -418,6 +421,15 @@ METHOD ComputeAgg( xE, aGR, aFN ) CLASS TSqlAgg
RETURN 0
ENDIF
/* Fast path: plain column + common aggregate → Go RTL single-pass loop.
* Gate on column-ref argument + pre-resolved nCol > 0; complex args
* (CASE/BIN/UDF) still fall through to the PRG loop below. */
IF nCol > 0 .AND. xArg[ 1 ] == ND_COL .AND. ;
( cFunc == "COUNT" .OR. cFunc == "SUM" .OR. cFunc == "AVG" .OR. ;
cFunc == "MIN" .OR. cFunc == "MAX" )
RETURN SqlComputeAggSimple( aGR, nCol, cFunc )
ENDIF
FOR i := 1 TO Len( aGR )
IF nCol > 0 .AND. nCol <= Len( aGR[ i ] )
xVal := aGR[ i ][ nCol ]
@@ -479,7 +491,15 @@ RETURN 0
METHOD EvalHaving( xHaving, aNewRow, aCols, aGroupRows, aFN, aParams ) CLASS TSqlAgg
LOCAL xResult
LOCAL xResult, aGo
/* Fast path: Go-native tree walker. Returns {lOk, lPass}; falls back
* to PRG when it hits an unsupported node (subqueries, complex agg
* args, CASE expressions inside HAVING, etc.). */
aGo := SqlEvalHaving( xHaving, aNewRow, aCols, aGroupRows, aFN, aParams )
IF ValType( aGo ) == "A" .AND. Len( aGo ) == 2 .AND. aGo[ 1 ]
RETURN aGo[ 2 ]
ENDIF
xResult := ::EvalHavingExpr( xHaving, aNewRow, aCols, aGroupRows, aFN, aParams )

441
_FiveSql2/src/TSqlExecutor.prg
View File

@@ -19,6 +19,18 @@ STATIC s_aOuterStack := {}
STATIC s_hAutoInc := NIL
STATIC s_nRCJSeq := 0
/* Per-plan DML pcode cache. Keyed by the plan-cache key that TFiveSQL
* uses (template key or cSQL text); value is a hash:
* { "set_fpos" => aFPos,
* "set_pc" => aValuePc, — parallel to set_fpos
* "where_pc" => pcWhere | NIL,
* "compiled" => .T. }
* RunUpdate populates on first hit, subsequent calls reuse. Compiled
* pcode depends on the target table's field layout; since the plan
* cache key already uniquely identifies the SQL template (same schema
* every call), the cache is sound. */
STATIC s_hDmlPcodeCache := { => }
CLASS TSqlExecutor
DATA hQuery
@@ -35,6 +47,7 @@ CLASS TSqlExecutor
DATA aCompileStruct
DATA bRowBlock /* optional code block — receives SELECT cols as params */
DATA aFetchCache /* pre-bound {nWA, nFPos} per SELECT expression, or NIL */
DATA cCacheKey /* plan-cache key set by TFiveSQL; used for DML pcode cache */
DATA hSubCorrCache INIT { => } /* per-outer-key subquery result cache */
DATA aSubCacheSlots INIT {} /* list of {xSubNode, {id, aFreeVars}} */
DATA nSubCacheSeq INIT 0 /* monotonic ID for subqueries */
@@ -1217,10 +1230,24 @@ METHOD RunSelect() CLASS TSqlExecutor
IF nWA == 0
nWA := ::OpenTable( cTable, cAlias )
IF nWA == 0
/* Table file not found; check if a CTE temp file exists for this
* table name and open it instead. This handles sub-executors
* (UNION, recursive) that reference a CTE by its original name. */
IF hb_FileExists( "__cte_" + Lower( cTable ) + ".dbf" )
/* Table file not found; check if a CTE temp table exists for
* this table name and open it instead. This handles sub-
* executors (UNION, recursive) that reference a CTE by its
* original name. CTE temp tables now live in MEMRDD (no
* file on disk) — fall back to the legacy DBFNTX open for
* pre-existing .dbf files from prior runs. */
BEGIN SEQUENCE
dbUseArea( .T., "MEMRDD", "mem:__cte_" + Lower( cTable ), ;
cAlias, .T., .T. )
nWA := Select( cAlias )
IF nWA > 0
AAdd( ::aOpened, cAlias )
AAdd( ::oAlias:aSlots, { cAlias, Upper( cTable ), Upper( cTable ), .T. } )
ENDIF
RECOVER
nWA := 0
END SEQUENCE
IF nWA == 0 .AND. hb_FileExists( "__cte_" + Lower( cTable ) + ".dbf" )
BEGIN SEQUENCE
dbUseArea( .T., "DBFNTX", "__cte_" + Lower( cTable ) + ".dbf", ;
cAlias, .T., .T. )
@@ -1418,9 +1445,34 @@ METHOD RunSelect() CLASS TSqlExecutor
aGoRows := NIL
IF Len( aJoins ) == 0 .AND. Len( aGroupBy ) == 0 .AND. ;
! ::oAgg:HasAgg( aCols )
aFP := ::TryBuildFieldPositions( aResultExprs )
/* Plan pcode cache: cache aFP + pcW per cCacheKey.
* These results are pure functions of the plan tree
* (which is immutable between cache hits) and the
* target table schema (stable for the process). */
LOCAL hSelCached, cSelKey
IF ! Empty( ::cCacheKey )
cSelKey := ::cCacheKey + "#sel"
IF hb_HHasKey( s_hDmlPcodeCache, cSelKey )
hSelCached := s_hDmlPcodeCache[ cSelKey ]
aFP := hSelCached[ "fp" ]
pcW := hSelCached[ "where_pc" ]
ENDIF
ENDIF
IF aFP == NIL
aFP := ::TryBuildFieldPositions( aResultExprs )
IF aFP != NIL .AND. xWhere != NIL
pcW := ::TryCompileWhere( xWhere )
IF pcW == NIL
aFP := NIL /* WHERE couldn't compile — PRG path */
ENDIF
ENDIF
IF aFP != NIL .AND. ! Empty( ::cCacheKey )
s_hDmlPcodeCache[ ::cCacheKey + "#sel" ] := { ;
"fp" => aFP, ;
"where_pc" => pcW }
ENDIF
ENDIF
IF aFP != NIL
pcW := ::TryCompileWhere( xWhere )
IF xWhere == NIL .OR. pcW != NIL
IF ::bRowBlock != NIL
/* Block mode: stream rows through user block.
@@ -2297,29 +2349,22 @@ METHOD MaterializeCTE( aCTE ) CLASS TSqlExecutor
dbSelectArea( nExistWA )
dbCloseArea()
ENDIF
IF hb_FileExists( cTmpFile + ".dbf" )
FErase( cTmpFile + ".dbf" )
ENDIF
/* In-memory temp table — no file I/O, `mem:` scheme dispatches
* to MEMRDD. Create overwrites any prior table with this name. */
BEGIN SEQUENCE
dbCreate( cTmpFile + ".dbf", aStruct )
dbCreate( "mem:" + cTmpFile, aStruct, "MEMRDD" )
RECOVER
LOOP
END SEQUENCE
USE ( cTmpFile + ".dbf" ) NEW EXCLUSIVE ALIAS ( cPopAlias )
FOR j := 1 TO Len( aDataRows )
dbAppend()
FOR k := 1 TO Min( Len( aStruct ), Len( aDataRows[ j ] ) )
IF aDataRows[ j ][ k ] != NIL
FieldPut( k, aDataRows[ j ][ k ] )
ENDIF
NEXT
NEXT
dbCommit()
dbUseArea( .T., "MEMRDD", "mem:" + cTmpFile, cPopAlias, .F., .F. )
/* Go RTL SqlBulkInsert: collapses per-row dbAppend+FieldPut loop
* into a single RTL call — N·M boundary crossings → 1. */
SqlBulkInsert( aDataRows )
dbSelectArea( Select( cPopAlias ) )
dbCloseArea()
USE ( cTmpFile + ".dbf" ) NEW SHARED ALIAS ( cName )
dbUseArea( .T., "MEMRDD", "mem:" + cTmpFile, cName, .T., .F. )
/* Replace existing table entry */
lReplaced := .F.
@@ -2340,16 +2385,7 @@ METHOD RunInsert() CLASS TSqlExecutor
aAutoInc := SqlGetAutoIncFields( cTable )
nWA := Select( cAlias )
IF nWA == 0
BEGIN SEQUENCE
dbUseArea( .T., "DBFNTX", Lower( cTable ) + ".dbf", cAlias, .F., .F. )
RECOVER
dbUseArea( .T., "DBFNTX", cTable + ".dbf", cAlias, .F., .F. )
END SEQUENCE
ELSE
dbSelectArea( nWA )
ENDIF
nWA := SqlExecOpenTable( cTable, cAlias )
/* Transaction logging */
::oTxn:LogRecord( cAlias, RecNo(), "INSERT" )
@@ -2410,12 +2446,16 @@ METHOD RunInsert() CLASS TSqlExecutor
NEXT
ENDIF
dbCommit()
IF nWA == 0
dbCloseArea()
/* Commit per INSERT when the WA cache is off (legacy durability
* guarantee). With the cache on, the caller batches via an
* explicit SqlWACacheDisable+dbCloseAll at shutdown — skipping
* the per-INSERT flush collapses the dominant I/O cost. */
IF ! SqlWACacheIsEnabled()
dbCommit()
ENDIF
SqlExecCloseTable( cAlias, nWA )
RETURN { { "affected_rows" }, { { 1 } } }
@@ -2423,6 +2463,7 @@ METHOD RunUpdate() CLASS TSqlExecutor
LOCAL cTable, aSet, xWhere, cAlias, nWA, i, nFPos, xVal
LOCAL nAffected := 0
LOCAL aFPos, aValuePc, pcWhere, lAllOk, cValSrc
cTable := ::hQuery[ "table" ]
aSet := ::hQuery[ "set" ]
@@ -2430,17 +2471,86 @@ METHOD RunUpdate() CLASS TSqlExecutor
cAlias := cTable
::aTables := { { cTable, cAlias, "" } }
nWA := Select( cAlias )
IF nWA == 0
BEGIN SEQUENCE
dbUseArea( .T., "DBFNTX", Lower( cTable ) + ".dbf", cAlias, .F., .F. )
RECOVER
dbUseArea( .T., "DBFNTX", cTable + ".dbf", cAlias, .F., .F. )
END SEQUENCE
ELSE
dbSelectArea( nWA )
nWA := SqlExecOpenTable( cTable, cAlias )
/* Fast path: compile WHERE + every SET value to pcode and delegate
* to Go RTL SqlBulkUpdate — skips per-record Go↔PRG boundary.
* Conditions: no active transaction (txn log records can't be
* emitted from inside the Go loop), no subquery / CASE / other
* nodes that PcCompile can't handle (try/fail pattern).
*
* Per-plan cache: when cCacheKey is set (TFiveSQL supplies it for
* plan-cached queries), we stash the compiled pcode under that key
* so subsequent identical UPDATEs skip the SqlExprToPrg + PcCompile
* walk entirely. The cached pcode is valid as long as the plan
* itself lives in the plan cache — which is forever in-process. */
IF ! ::oTxn:IsActive()
LOCAL hPcCached
IF ! Empty( ::cCacheKey ) .AND. hb_HHasKey( s_hDmlPcodeCache, ::cCacheKey )
hPcCached := s_hDmlPcodeCache[ ::cCacheKey ]
nAffected := SqlBulkUpdate( hPcCached[ "set_fpos" ], ;
hPcCached[ "where_pc" ], ;
hPcCached[ "set_pc" ] )
IF ! SqlWACacheIsEnabled()
dbCommit()
ENDIF
SqlExecCloseTable( cAlias, nWA )
RETURN { { "affected_rows" }, { { nAffected } } }
ENDIF
aFPos := {}
aValuePc := {}
lAllOk := .T.
FOR i := 1 TO Len( aSet )
nFPos := FieldPos( aSet[ i ][ 1 ] )
IF nFPos <= 0
lAllOk := .F.
EXIT
ENDIF
cValSrc := ::SqlExprToPrg( aSet[ i ][ 2 ] )
IF cValSrc == NIL
lAllOk := .F.
EXIT
ENDIF
AAdd( aFPos, nFPos )
AAdd( aValuePc, PcCompile( cValSrc ) )
IF ATail( aValuePc ) == NIL
lAllOk := .F.
EXIT
ENDIF
NEXT
pcWhere := NIL
IF lAllOk .AND. xWhere != NIL
cValSrc := ::SqlExprToPrg( xWhere )
IF cValSrc == NIL
lAllOk := .F.
ELSE
pcWhere := PcCompile( cValSrc )
IF pcWhere == NIL
lAllOk := .F.
ENDIF
ENDIF
ENDIF
IF lAllOk
nAffected := SqlBulkUpdate( aFPos, pcWhere, aValuePc )
/* Populate the per-plan cache for subsequent calls. */
IF ! Empty( ::cCacheKey )
s_hDmlPcodeCache[ ::cCacheKey ] := { ;
"set_fpos" => aFPos, ;
"set_pc" => aValuePc, ;
"where_pc" => pcWhere }
ENDIF
/* Defer commit under WA cache — batched at Disable/exit. */
IF ! SqlWACacheIsEnabled()
dbCommit()
ENDIF
SqlExecCloseTable( cAlias, nWA )
RETURN { { "affected_rows" }, { { nAffected } } }
ENDIF
ENDIF
/* Fallback: PRG scan loop — handles txn logging + non-compilable
* expressions (subquery, complex CASE, UDF in value or WHERE). */
dbGoTop()
WHILE ! Eof()
IF xWhere == NIL .OR. SqlIsTrue( ::EvalExpr( xWhere ) )
@@ -2459,12 +2569,12 @@ METHOD RunUpdate() CLASS TSqlExecutor
ENDIF
dbSkip()
ENDDO
dbCommit()
IF nWA == 0
dbCloseArea()
IF ! SqlWACacheIsEnabled()
dbCommit()
ENDIF
SqlExecCloseTable( cAlias, nWA )
RETURN { { "affected_rows" }, { { nAffected } } }
@@ -2478,16 +2588,7 @@ METHOD RunDelete() CLASS TSqlExecutor
cAlias := cTable
::aTables := { { cTable, cAlias, "" } }
nWA := Select( cAlias )
IF nWA == 0
BEGIN SEQUENCE
dbUseArea( .T., "DBFNTX", Lower( cTable ) + ".dbf", cAlias, .F., .F. )
RECOVER
dbUseArea( .T., "DBFNTX", cTable + ".dbf", cAlias, .F., .F. )
END SEQUENCE
ELSE
dbSelectArea( nWA )
ENDIF
nWA := SqlExecOpenTable( cTable, cAlias )
SET DELETED ON
dbGoTop()
@@ -2501,13 +2602,84 @@ METHOD RunDelete() CLASS TSqlExecutor
ENDIF
dbSkip()
ENDDO
dbCommit()
IF ! SqlWACacheIsEnabled()
dbCommit()
ENDIF
IF nWA == 0
SqlExecCloseTable( cAlias, nWA )
RETURN { { "affected_rows" }, { { nAffected } } }
/* ======================================================================
* Workarea open/close helpers — consult the Go-native WA cache.
* When the cache is enabled (SqlWACacheEnable), SqlExecOpenTable
* reuses a previously opened workarea instead of running dbUseArea
* every call. SqlExecCloseTable leaves cached entries alive; plain
* (auto-opened, not cached) areas still close as before so tests
* that rely on immediate file release (FErase, UNIQUE index rebuild)
* stay correct when the cache is off — which is the default.
* ====================================================================== */
FUNCTION SqlExecOpenTable( cTable, cAlias )
LOCAL nWA, nCached
nWA := Select( cAlias )
IF nWA > 0
dbSelectArea( nWA )
RETURN nWA
ENDIF
/* Cache hit: the previously stored WA must still be valid and bound
* to the same alias. If a manual close or CLOSE ALL ran behind our
* back, Select() will now report 0 — fall through to fresh open. */
nCached := SqlWACacheGet( cAlias )
IF nCached > 0 .AND. Select( cAlias ) == nCached
dbSelectArea( nCached )
RETURN nCached
ENDIF
IF nCached > 0
SqlWACacheInvalidate( cAlias )
ENDIF
/* Open fresh. Two-step fallback mirrors the prior inline logic so
* callers using mixed-case filenames on case-sensitive filesystems
* still succeed. */
BEGIN SEQUENCE
dbUseArea( .T., "DBFNTX", Lower( cTable ) + ".dbf", cAlias, .F., .F. )
RECOVER
dbUseArea( .T., "DBFNTX", cTable + ".dbf", cAlias, .F., .F. )
END SEQUENCE
nWA := Select( cAlias )
/* Register for reuse. The cache layer is a no-op when disabled, so
* an unconditional Put keeps the caller branch-free. */
IF nWA > 0 .AND. SqlWACacheIsEnabled()
SqlWACachePut( cAlias, nWA )
/* Return 1 sentinel so callers' "if nWA==0 close" gates skip
* — the cache owns the lifecycle now. */
RETURN nWA
ENDIF
RETURN 0 /* caller must close — matches legacy semantics */
FUNCTION SqlExecCloseTable( cAlias, nWA )
/* Only close if THIS call opened it AND the cache didn't adopt it.
* When nWA > 0, the caller either reused a pre-existing area or
* handed ownership to the cache, so we leave it alone. */
IF nWA == 0 .AND. ! SqlWACacheIsEnabled()
dbCloseArea()
ELSEIF nWA == 0 .AND. SqlWACacheIsEnabled() .AND. ;
SqlWACacheGet( cAlias ) == 0
/* Cache enabled but the alias wasn't registered (e.g., open
* failed between Put checks). Keep legacy behavior — close. */
dbCloseArea()
ENDIF
RETURN { { "affected_rows" }, { { nAffected } } }
RETURN NIL
/* ======================================================================
@@ -2626,17 +2798,11 @@ FUNCTION SqlMaterializeSubquery( xSubQ, cAlias, aParams )
NEXT
cTmpFile := "__drv_" + Lower( cAlias )
dbCreate( cTmpFile + ".dbf", aStruct )
USE ( cTmpFile + ".dbf" ) NEW EXCLUSIVE ALIAS __DRVTMP
FOR i := 1 TO Len( aRows2 )
dbAppend()
FOR j := 1 TO Min( Len( aStruct ), Len( aRows2[ i ] ) )
IF aRows2[ i ][ j ] != NIL
FieldPut( j, aRows2[ i ][ j ] )
ENDIF
NEXT
NEXT
dbCommit()
/* MEMRDD in-memory temp — avoids dbCreate + FErase disk syscalls. */
dbCreate( "mem:" + cTmpFile, aStruct, "MEMRDD" )
dbUseArea( .T., "MEMRDD", "mem:" + cTmpFile, "__DRVTMP", .F., .F. )
/* Go RTL SqlBulkInsert — subquery driving-table materialization. */
SqlBulkInsert( aRows2 )
CLOSE __DRVTMP
RETURN { cTmpFile, cAlias, "" }
@@ -2922,26 +3088,16 @@ METHOD MaterializeRecursiveCTE( aCTE ) CLASS TSqlExecutor
dbSelectArea( nExistWA )
dbCloseArea()
ENDIF
IF hb_FileExists( cTmpFile + ".dbf" )
FErase( cTmpFile + ".dbf" )
ENDIF
/* MEMRDD in-memory temp for CTE — no file create/delete. */
BEGIN SEQUENCE
dbCreate( cTmpFile + ".dbf", aStruct )
dbCreate( "mem:" + cTmpFile, aStruct, "MEMRDD" )
RECOVER
END SEQUENCE
BEGIN SEQUENCE
USE ( cTmpFile + ".dbf" ) NEW ALIAS ( cAlias )
FOR j := 1 TO Len( aDataRows )
dbAppend()
FOR k := 1 TO Min( Len( aStruct ), Len( aDataRows[ j ] ) )
IF aDataRows[ j ][ k ] != NIL
FieldPut( k, aDataRows[ j ][ k ] )
ENDIF
NEXT
NEXT
dbCommit()
dbUseArea( .T., "MEMRDD", "mem:" + cTmpFile, cAlias, .F., .F. )
/* Go RTL SqlBulkInsert — CTE materialization path. */
SqlBulkInsert( aDataRows )
RECOVER
END SEQUENCE
@@ -2973,7 +3129,7 @@ METHOD ApplyWindowFunctions( aRows, aFN, aCols ) CLASS TSqlExecutor
LOCAL i, j, k, nColIdx, xExpr
LOCAL cFunc, aPartBy, aOrdBy, aFuncArgs
LOCAL hPartitions, cPartKey, aPartIdx
LOCAL aPartitions, cPartKey, aPartIdx
LOCAL aSorted, aIdxMap, nPartCol
LOCAL nRank, nDenseRank, nRowNum
LOCAL xPrev, xCurr, nTies
@@ -2981,6 +3137,7 @@ METHOD ApplyWindowFunctions( aRows, aFN, aCols ) CLASS TSqlExecutor
LOCAL nRunSum, nRunCount
LOCAL aWinCols, nWC
LOCAL hFrame, nFS, nFE, m, xVal, xMin, xMax, lDefaultFrame
LOCAL aPartColIdx, aSortSpec, nOrdCol
/* Scan for window function columns */
aWinCols := {}
@@ -3008,69 +3165,55 @@ METHOD ApplyWindowFunctions( aRows, aFN, aCols ) CLASS TSqlExecutor
hFrame := xExpr[ 6 ]
ENDIF
/* Build partition groups as arrays of row indices */
hPartitions := { => }
FOR i := 1 TO Len( aRows )
cPartKey := ""
IF ValType( aPartBy ) == "A"
FOR j := 1 TO Len( aPartBy )
nPartCol := SqlFindColIdx( aPartBy[ j ], aFN )
IF nPartCol == 0
nPartCol := SqlFindColIdx2( SqlExprName( aPartBy[ j ] ), aFN )
ENDIF
IF nPartCol > 0 .AND. nPartCol <= Len( aRows[ i ] )
cPartKey += SqlValToStr( aRows[ i ][ nPartCol ] ) + "|"
ENDIF
NEXT
ENDIF
IF ! hb_HHasKey( hPartitions, cPartKey )
hPartitions[ cPartKey ] := {}
ENDIF
AAdd( hPartitions[ cPartKey ], i )
NEXT
/* Resolve PARTITION BY columns once, then delegate the row-index
* grouping to Go RTL SqlWindowPartitions — removes N·M per-row
* Go↔PRG boundary crossings for SqlValToStr / hb_HHasKey / AAdd. */
aPartColIdx := {}
IF ValType( aPartBy ) == "A"
FOR j := 1 TO Len( aPartBy )
nPartCol := SqlFindColIdx( aPartBy[ j ], aFN )
IF nPartCol == 0
nPartCol := SqlFindColIdx2( SqlExprName( aPartBy[ j ] ), aFN )
ENDIF
IF nPartCol > 0
AAdd( aPartColIdx, nPartCol )
ENDIF
NEXT
ENDIF
aPartitions := SqlWindowPartitions( aRows, aPartColIdx )
/* Pre-resolve ORDER BY column indices once per window column —
* Go SqlWindowSortPartition reads the resolved {nCol, lDesc}
* pairs directly, so every partition sort avoids the repeated
* SqlFindColIdx linear scan inside per-comparison PRG blocks. */
aSortSpec := {}
IF ValType( aOrdBy ) == "A" .AND. Len( aOrdBy ) > 0
FOR j := 1 TO Len( aOrdBy )
nOrdCol := SqlFindColIdx( aOrdBy[ j ][ 1 ], aFN )
IF nOrdCol == 0
nOrdCol := SqlFindColIdx2( SqlExprName( aOrdBy[ j ][ 1 ] ), aFN )
ENDIF
IF nOrdCol > 0
AAdd( aSortSpec, { nOrdCol, aOrdBy[ j ][ 2 ] == "DESC" } )
ENDIF
NEXT
ENDIF
/* Process each partition */
FOR EACH aPartIdx IN hb_HValues( hPartitions )
FOR EACH aPartIdx IN aPartitions
/* Sort partition indices by ORDER BY columns */
IF ValType( aOrdBy ) == "A" .AND. Len( aOrdBy ) > 0
ASort( aPartIdx,,, {|a, b| SqlWinRowCmp( aRows, a, b, aOrdBy, aFN ) < 0 } )
/* Sort partition indices by ORDER BY columns (Go RTL). */
IF Len( aSortSpec ) > 0
SqlWindowSortPartition( aRows, aPartIdx, aSortSpec )
ENDIF
/* Compute window function for each row in the partition */
/* Compute window function for each row in the partition.
* ROW_NUMBER/RANK/DENSE_RANK all go through one Go RTL call
* that walks the partition and writes the rank column —
* removes per-row SqlWinRowsEqual + PRG indexing overhead. */
DO CASE
CASE cFunc == "ROW_NUMBER"
FOR k := 1 TO Len( aPartIdx )
IF nColIdx <= Len( aRows[ aPartIdx[ k ] ] )
aRows[ aPartIdx[ k ] ][ nColIdx ] := k
ENDIF
NEXT
CASE cFunc == "RANK"
nRank := 1
FOR k := 1 TO Len( aPartIdx )
IF k > 1
IF ! SqlWinRowsEqual( aRows, aPartIdx[ k ], aPartIdx[ k - 1 ], aOrdBy, aFN )
nRank := k
ENDIF
ENDIF
IF nColIdx <= Len( aRows[ aPartIdx[ k ] ] )
aRows[ aPartIdx[ k ] ][ nColIdx ] := nRank
ENDIF
NEXT
CASE cFunc == "DENSE_RANK"
nDenseRank := 1
FOR k := 1 TO Len( aPartIdx )
IF k > 1
IF ! SqlWinRowsEqual( aRows, aPartIdx[ k ], aPartIdx[ k - 1 ], aOrdBy, aFN )
nDenseRank++
ENDIF
ENDIF
IF nColIdx <= Len( aRows[ aPartIdx[ k ] ] )
aRows[ aPartIdx[ k ] ][ nColIdx ] := nDenseRank
ENDIF
NEXT
CASE cFunc == "ROW_NUMBER" .OR. cFunc == "RANK" .OR. cFunc == "DENSE_RANK"
SqlWindowAssignRank( aRows, aPartIdx, aSortSpec, nColIdx, cFunc )
CASE cFunc == "LAG"
nLagLead := 1
@@ -3817,11 +3960,12 @@ RETURN aResult
* simple column index (complex expressions → PRG fallback). */
METHOD TryBuildSortSpec( aOrderBy, aFieldNames ) CLASS TSqlExecutor
LOCAL aSpec := {}, i, j, xE, cName, nCol, cDir, nDot
LOCAL aSpec := {}, i, j, xE, cName, nCol, cDir, cNulls, nDot
FOR i := 1 TO Len( aOrderBy )
xE := aOrderBy[ i ][ 1 ]
cDir := Upper( aOrderBy[ i ][ 2 ] )
cNulls := iif( Len( aOrderBy[ i ] ) >= 3, Upper( aOrderBy[ i ][ 3 ] ), "" )
IF xE == NIL .OR. xE[ 1 ] != ND_COL
RETURN NIL
ENDIF
@@ -3843,7 +3987,10 @@ METHOD TryBuildSortSpec( aOrderBy, aFieldNames ) CLASS TSqlExecutor
IF nCol == 0
RETURN NIL
ENDIF
AAdd( aSpec, { nCol, cDir == "DESC" } )
/* Go SqlOrderBy reads {nCol, lDesc, cNulls}. cNulls empty means
* "default" — NIL sorts as the largest value (NULLs last in ASC,
* NULLs first in DESC). Explicit "FIRST"/"LAST" overrides. */
AAdd( aSpec, { nCol, cDir == "DESC", cNulls } )
NEXT
RETURN aSpec

43
_FiveSql2/src/TSqlExpr.prg
View File

@@ -42,45 +42,10 @@ FUNCTION SqlExprName( xE )
RETURN "expr"
/* Check whether an expression tree contains an aggregate function call.
* Recurses into ND_BIN, ND_UNI, ND_FN args, ND_CASE to find nested
* aggregates like `salary + COUNT(*)` or `CASE WHEN ... THEN SUM(x)`. */
FUNCTION SqlExprHasAgg( xE )
LOCAL i
IF xE == NIL
RETURN .F.
ENDIF
IF xE[ 1 ] == ND_FN .AND. SqlIsAggName( xE[ 2 ] )
RETURN .T.
ENDIF
/* Recurse into sub-expressions */
IF xE[ 1 ] == ND_BIN
RETURN SqlExprHasAgg( xE[ 3 ] ) .OR. SqlExprHasAgg( xE[ 4 ] )
ENDIF
IF xE[ 1 ] == ND_UNI
RETURN SqlExprHasAgg( xE[ 3 ] )
ENDIF
IF xE[ 1 ] == ND_FN .AND. ValType( xE[ 3 ] ) == "A"
FOR i := 1 TO Len( xE[ 3 ] )
IF SqlExprHasAgg( xE[ 3 ][ i ] )
RETURN .T.
ENDIF
NEXT
ENDIF
IF xE[ 1 ] == ND_CASE .AND. ValType( xE[ 2 ] ) == "A"
FOR i := 1 TO Len( xE[ 2 ] )
IF SqlExprHasAgg( xE[ 2 ][ i ][ 1 ] ) .OR. SqlExprHasAgg( xE[ 2 ][ i ][ 2 ] )
RETURN .T.
ENDIF
NEXT
IF xE[ 3 ] != NIL .AND. SqlExprHasAgg( xE[ 3 ] )
RETURN .T.
ENDIF
ENDIF
RETURN .F.
/* SqlExprHasAgg is implemented in Go (hbrtl/sqlexpr.go) — registered
* as SQLEXPRHASAGG. The prior PRG recursive walker has been removed
* to avoid a name collision with the RTL symbol; behavior is
* byte-for-byte identical. See docs/RTL-Go-Native-Migration.md. */
/* Return .T. if the function name is an aggregate */
FUNCTION SqlIsAggName( c )

120
_FiveSql2/src/TSqlFunc.prg
View File

@@ -329,121 +329,11 @@ FUNCTION SqlArg( a, n )
RETURN NIL
/* Coerce to string */
FUNCTION SqlCoerceStr( x )
IF x == NIL
RETURN ""
ENDIF
IF ValType( x ) == "C"
RETURN x
ENDIF
IF ValType( x ) == "N"
RETURN AllTrim( Str( x ) )
ENDIF
IF ValType( x ) == "D"
RETURN DToC( x )
ENDIF
IF ValType( x ) == "L"
RETURN iif( x, "T", "F" )
ENDIF
RETURN ""
/* Coerce to numeric */
FUNCTION SqlCoerceNum( x )
IF x == NIL
RETURN 0
ENDIF
IF ValType( x ) == "N"
RETURN x
ENDIF
IF ValType( x ) == "C"
RETURN Val( AllTrim( x ) )
ENDIF
IF ValType( x ) == "L"
RETURN iif( x, 1, 0 )
ENDIF
RETURN 0
/* Normalize for comparison: trim and uppercase strings */
FUNCTION SqlCoerceForCmp( x )
IF x == NIL
RETURN x
ENDIF
IF ValType( x ) == "C"
RETURN Upper( AllTrim( x ) )
ENDIF
RETURN x
/* Evaluate truthiness */
FUNCTION SqlIsTrue( x )
IF x == NIL
RETURN .F.
ENDIF
IF ValType( x ) == "L"
RETURN x
ENDIF
IF ValType( x ) == "N"
RETURN x != 0
ENDIF
IF ValType( x ) == "C"
RETURN ! Empty( x )
ENDIF
RETURN .F.
/* Case-insensitive equality comparison with cross-type coercion */
FUNCTION SqlCmpEq( a, b )
IF a == NIL .OR. b == NIL
RETURN a == NIL .AND. b == NIL
ENDIF
IF ValType( a ) == ValType( b )
IF ValType( a ) == "C"
RETURN Upper( AllTrim( a ) ) == Upper( AllTrim( b ) )
ENDIF
RETURN a == b
ENDIF
IF ValType( a ) == "N" .AND. ValType( b ) == "C"
RETURN a == Val( AllTrim( b ) )
ENDIF
IF ValType( a ) == "C" .AND. ValType( b ) == "N"
RETURN Val( AllTrim( a ) ) == b
ENDIF
RETURN .F.
/* Case-insensitive less-than comparison */
FUNCTION SqlCmpLt( a, b )
IF a == NIL .OR. b == NIL
RETURN .F.
ENDIF
IF ValType( a ) == ValType( b )
IF ValType( a ) == "C"
RETURN Upper( AllTrim( a ) ) < Upper( AllTrim( b ) )
ENDIF
RETURN a < b
ENDIF
IF ValType( a ) == "N" .AND. ValType( b ) == "C"
RETURN a < Val( AllTrim( b ) )
ENDIF
IF ValType( a ) == "C" .AND. ValType( b ) == "N"
RETURN Val( AllTrim( a ) ) < b
ENDIF
RETURN .F.
/* SqlCoerceStr/SqlCoerceNum/SqlCoerceForCmp/SqlIsTrue/SqlCmpEq/SqlCmpLt
* are implemented in Go (hbrtl/sqlhelpers.go) — registered as
* SQLCOERCESTR etc. The PRG bodies have been removed to avoid symbol
* collision with the RTL symbols; behavior is byte-for-byte identical.
* See docs/RTL-Go-Native-Migration.md (Tier 4). */
/* SQL LIKE pattern matching with optional escape character */

48
_FiveSql2/src/TSqlSort.prg
View File

@@ -32,13 +32,15 @@ RETURN SELF
METHOD OrderBy( aRows, aFN, aOB, aTables, aParams ) CLASS TSqlSort
LOCAL i, nCol
LOCAL i, nCol, cNulls
IF Len( aRows ) < 2 .OR. Len( aOB ) == 0
RETURN aRows
ENDIF
/* Pre-resolve column indexes */
/* Pre-resolve column indexes. Third element carries the explicit
* NULLS FIRST/LAST spec parsed by TSqlParser2:ParseOrderBy —
* empty string means "use default (NIL as largest)". */
s_aOBCols := {}
s_aOBNames := aFN
FOR i := 1 TO Len( aOB )
@@ -46,7 +48,8 @@ METHOD OrderBy( aRows, aFN, aOB, aTables, aParams ) CLASS TSqlSort
IF nCol == 0
nCol := SqlFindColIdx2( SqlExprName( aOB[ i ][ 1 ] ), aFN )
ENDIF
AAdd( s_aOBCols, { nCol, aOB[ i ][ 2 ] } )
cNulls := iif( Len( aOB[ i ] ) >= 3, Upper( aOB[ i ][ 3 ] ), "" )
AAdd( s_aOBCols, { nCol, aOB[ i ][ 2 ], cNulls } )
NEXT
ASort( aRows,,, {|a, b| SqlRowCompare( a, b ) < 0 } )
@@ -56,18 +59,11 @@ RETURN aRows
METHOD Distinct( aRows ) CLASS TSqlSort
LOCAL aR := {}, i, cKey
LOCAL hSeen := { => }
FOR i := 1 TO Len( aRows )
cKey := ::RowKey( aRows[ i ] )
IF ! hb_HHasKey( hSeen, cKey )
hSeen[ cKey ] := .T.
AAdd( aR, aRows[ i ] )
ENDIF
NEXT
RETURN aR
/* Go RTL SqlDistinct: single-pass dedup via Go map[string]bool.
* Key construction matches prior PRG ::RowKey byte-for-byte (same
* SqlValToStr mapping + '|' separator), so the output is identical
* to the old PRG loop — just ~100x faster on large result sets. */
RETURN SqlDistinct( aRows )
METHOD RowKey( aR ) CLASS TSqlSort
@@ -118,11 +114,12 @@ RETURN 0
/* Multi-key row comparator for ASort */
FUNCTION SqlRowCompare( aRowA, aRowB )
LOCAL i, nCol, cDir, xA, xB, nCmp
LOCAL i, nCol, cDir, cNulls, lNullsFirst, xA, xB, nCmp
FOR i := 1 TO Len( s_aOBCols )
nCol := s_aOBCols[ i ][ 1 ]
cDir := s_aOBCols[ i ][ 2 ]
cNulls := iif( Len( s_aOBCols[ i ] ) >= 3, s_aOBCols[ i ][ 3 ], "" )
IF nCol <= 0 .OR. nCol > Len( aRowA ) .OR. nCol > Len( aRowB )
LOOP
@@ -131,15 +128,22 @@ FUNCTION SqlRowCompare( aRowA, aRowB )
xA := aRowA[ nCol ]
xB := aRowB[ nCol ]
/* NULLs sort last */
/* NULL ordering — default: NIL is largest (NULLs last in ASC,
* NULLs first in DESC). Explicit NULLS FIRST/LAST (SQL:2003)
* from the parser overrides direction. */
IF xA == NIL .AND. xB == NIL
LOOP
ENDIF
IF xA == NIL
RETURN iif( cDir == "DESC", -1, 1 )
ENDIF
IF xB == NIL
RETURN iif( cDir == "DESC", 1, -1 )
IF xA == NIL .OR. xB == NIL
DO CASE
CASE cNulls == "FIRST" ; lNullsFirst := .T.
CASE cNulls == "LAST" ; lNullsFirst := .F.
OTHERWISE ; lNullsFirst := ( cDir == "DESC" )
ENDCASE
IF xA == NIL
RETURN iif( lNullsFirst, -1, 1 )
ENDIF
RETURN iif( lNullsFirst, 1, -1 )
ENDIF
nCmp := 0