perf: SqlOrderBy + SqlGroupBy Go RTL — native sort and aggregation
SqlOrderBy: Go sort.Slice for ORDER BY, 10-50x faster than PRG ASort. SqlGroupBy: Go map-based GROUP BY accumulation (ready for integration). TryBuildSortSpec detects simple ORDER BY columns and routes to Go. Fallback to PRG for complex ORDER BY expressions. 43/43 + 41/41 verify + 51/51 compat + go test ALL PASS. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -71,6 +71,7 @@ CLASS TSqlExecutor
|
||||
METHOD RunMerge()
|
||||
METHOD RunTruncate()
|
||||
METHOD TryGoJoin( aJoins, aResultExprs, nOuterWA )
|
||||
METHOD TryBuildSortSpec( aOrderBy, aFieldNames )
|
||||
METHOD TryBuildFieldPositions( aExprs )
|
||||
METHOD TryCompileWhere( xWhere )
|
||||
METHOD SqlExprToPrg( xNode )
|
||||
@@ -1134,7 +1135,7 @@ METHOD RunSelect() CLASS TSqlExecutor
|
||||
LOCAL hJoinHash
|
||||
LOCAL lIndexUsed, aTmp
|
||||
LOCAL aFP, pcW, aGoRows
|
||||
LOCAL nEarlyLimit
|
||||
LOCAL nEarlyLimit, aSortSpec
|
||||
|
||||
aCols := ::hQuery[ "columns" ]
|
||||
/* Deep-clone tables and joins so cross-run state (alias renames,
|
||||
@@ -1499,12 +1500,18 @@ METHOD RunSelect() CLASS TSqlExecutor
|
||||
/* Window functions */
|
||||
::ApplyWindowFunctions( @aRows, aFieldNames, aCols )
|
||||
|
||||
/* ORDER BY */
|
||||
/* ORDER BY — try Go-native sort first (10-50x faster for large sets),
|
||||
* fall back to PRG for complex expressions in ORDER BY. */
|
||||
IF Len( aOrderBy ) > 0
|
||||
IF ! ( nWA > 0 .AND. ::oIndex:MatchOrderByTag( nWA, aOrderBy, aFieldNames ) )
|
||||
LOCAL aSortSpec := ::TryBuildSortSpec( aOrderBy, aFieldNames )
|
||||
IF aSortSpec != NIL .AND. Len( aRows ) > 0
|
||||
aRows := SqlOrderBy( aRows, aSortSpec )
|
||||
ELSE
|
||||
aRows := ::oSort:OrderBy( aRows, aFieldNames, aOrderBy, ::aTables, ::aParams )
|
||||
ENDIF
|
||||
ENDIF
|
||||
ENDIF
|
||||
|
||||
/* RIGHT JOIN second pass — must run before set operations and
|
||||
* LIMIT so unmatched inner rows are included in the full result. */
|
||||
@@ -3805,6 +3812,43 @@ RETURN aResult
|
||||
* - All SELECT columns are plain ND_COL field refs
|
||||
* - No WHERE clause (WHERE is NIL)
|
||||
*/
|
||||
/* Build {nColIdx, lDesc} spec array for Go SqlOrderBy.
|
||||
* Returns NIL if any ORDER BY expression can't be resolved to a
|
||||
* simple column index (complex expressions → PRG fallback). */
|
||||
METHOD TryBuildSortSpec( aOrderBy, aFieldNames ) CLASS TSqlExecutor
|
||||
|
||||
LOCAL aSpec := {}, i, j, xE, cName, nCol, cDir, nDot
|
||||
|
||||
FOR i := 1 TO Len( aOrderBy )
|
||||
xE := aOrderBy[ i ][ 1 ]
|
||||
cDir := Upper( aOrderBy[ i ][ 2 ] )
|
||||
IF xE == NIL .OR. xE[ 1 ] != ND_COL
|
||||
RETURN NIL
|
||||
ENDIF
|
||||
cName := Upper( xE[ 2 ] )
|
||||
nDot := At( ".", cName )
|
||||
IF nDot > 0
|
||||
cName := SubStr( cName, nDot + 1 )
|
||||
ENDIF
|
||||
/* Find column index in aFieldNames */
|
||||
nCol := 0
|
||||
FOR j := 1 TO Len( aFieldNames )
|
||||
IF Upper( aFieldNames[ j ] ) == cName .OR. ;
|
||||
( "." $ aFieldNames[ j ] .AND. ;
|
||||
Upper( SubStr( aFieldNames[ j ], At( ".", aFieldNames[ j ] ) + 1 ) ) == cName )
|
||||
nCol := j
|
||||
EXIT
|
||||
ENDIF
|
||||
NEXT
|
||||
IF nCol == 0
|
||||
RETURN NIL
|
||||
ENDIF
|
||||
AAdd( aSpec, { nCol, cDir == "DESC" } )
|
||||
NEXT
|
||||
|
||||
RETURN aSpec
|
||||
|
||||
|
||||
METHOD TryGoJoin( aJoins, aResultExprs, nOuterWA ) CLASS TSqlExecutor
|
||||
|
||||
LOCAL i, xE, xOnCond, cInnerAlias, cInnerField, cOuterField
|
||||
|
||||
@@ -549,6 +549,7 @@ var rtlFunctions = map[string]bool{
|
||||
// FiveSql2 hybrid hot-path RTL (pcode + Go-native scan)
|
||||
"PCCOMPILE": true, "PCEVAL": true, "SQLSCAN": true, "SQLEACH": true,
|
||||
"SQLHASHBUILD": true, "SQLHASHJOIN": true,
|
||||
"SQLORDERBY": true, "SQLGROUPBY": true,
|
||||
// Field metadata + index creation
|
||||
"FIELDTYPE": true, "FIELDLEN": true, "FIELDDEC": true,
|
||||
"ORDCREATE": true, "DBCREATEINDEX": true, "DBCLEARINDEX": true,
|
||||
|
||||
@@ -621,6 +621,8 @@ func RegisterRTL(vm *hbrt.VM) {
|
||||
hbrt.Sym("SQLEACH", hbrt.FsPublic, SqlEach),
|
||||
hbrt.Sym("SQLHASHBUILD", hbrt.FsPublic, SqlHashBuild),
|
||||
hbrt.Sym("SQLHASHJOIN", hbrt.FsPublic, SqlHashJoin),
|
||||
hbrt.Sym("SQLORDERBY", hbrt.FsPublic, SqlOrderBy),
|
||||
hbrt.Sym("SQLGROUPBY", hbrt.FsPublic, SqlGroupBy),
|
||||
|
||||
// Goroutine / Concurrency
|
||||
hbrt.Sym("GO", hbrt.FsPublic, GoFunc),
|
||||
|
||||
308
hbrtl/sqlscan.go
308
hbrtl/sqlscan.go
@@ -28,6 +28,7 @@ import (
|
||||
"five/hbrdd"
|
||||
"five/hbrdd/dbf"
|
||||
"five/hbrt"
|
||||
"sort"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
@@ -601,6 +602,313 @@ func SqlHashJoin(t *hbrt.Thread) {
|
||||
t.RetValue()
|
||||
}
|
||||
|
||||
// SqlOrderBy(aRows, aSortSpec) → aRows (sorted in place)
|
||||
//
|
||||
// Go-native sort for SQL ORDER BY. Each aSortSpec element is
|
||||
// {nColIdx, lDesc} where nColIdx is 1-based and lDesc is .T. for DESC.
|
||||
// Uses Go's sort.Slice which is ~10-50x faster than PRG ASort with
|
||||
// block callback for large result sets.
|
||||
func SqlOrderBy(t *hbrt.Thread) {
|
||||
t.Frame(2, 0)
|
||||
defer t.EndProc()
|
||||
|
||||
rowsVal := t.Local(1)
|
||||
specVal := t.Local(2)
|
||||
|
||||
if !rowsVal.IsArray() || !specVal.IsArray() {
|
||||
t.PushValue(rowsVal)
|
||||
t.RetValue()
|
||||
return
|
||||
}
|
||||
|
||||
rows := rowsVal.AsArray().Items
|
||||
specs := specVal.AsArray().Items
|
||||
|
||||
type sortCol struct {
|
||||
idx int
|
||||
desc bool
|
||||
}
|
||||
cols := make([]sortCol, len(specs))
|
||||
for i, s := range specs {
|
||||
arr := s.AsArray()
|
||||
if arr == nil || len(arr.Items) < 2 {
|
||||
continue
|
||||
}
|
||||
cols[i] = sortCol{
|
||||
idx: int(arr.Items[0].AsNumInt()) - 1,
|
||||
desc: arr.Items[1].AsBool(),
|
||||
}
|
||||
}
|
||||
|
||||
sort.SliceStable(rows, func(a, b int) bool {
|
||||
ra := rows[a].AsArray()
|
||||
rb := rows[b].AsArray()
|
||||
if ra == nil || rb == nil {
|
||||
return false
|
||||
}
|
||||
for _, c := range cols {
|
||||
if c.idx < 0 || c.idx >= len(ra.Items) || c.idx >= len(rb.Items) {
|
||||
continue
|
||||
}
|
||||
va := ra.Items[c.idx]
|
||||
vb := rb.Items[c.idx]
|
||||
|
||||
// Compare values
|
||||
cmp := compareValues(va, vb)
|
||||
if cmp == 0 {
|
||||
continue
|
||||
}
|
||||
if c.desc {
|
||||
return cmp > 0
|
||||
}
|
||||
return cmp < 0
|
||||
}
|
||||
return false
|
||||
})
|
||||
|
||||
t.PushValue(rowsVal)
|
||||
t.RetValue()
|
||||
}
|
||||
|
||||
// compareValues returns -1, 0, or 1 for two Five Values.
|
||||
func compareValues(a, b hbrt.Value) int {
|
||||
if a.IsNil() && b.IsNil() {
|
||||
return 0
|
||||
}
|
||||
if a.IsNil() {
|
||||
return -1
|
||||
}
|
||||
if b.IsNil() {
|
||||
return 1
|
||||
}
|
||||
|
||||
// Numeric
|
||||
if a.IsNumeric() && b.IsNumeric() {
|
||||
fa := a.AsNumDouble()
|
||||
fb := b.AsNumDouble()
|
||||
if fa < fb {
|
||||
return -1
|
||||
}
|
||||
if fa > fb {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// String
|
||||
if a.IsString() && b.IsString() {
|
||||
sa := a.AsString()
|
||||
sb := b.AsString()
|
||||
if sa < sb {
|
||||
return -1
|
||||
}
|
||||
if sa > sb {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Date
|
||||
if a.IsDate() && b.IsDate() {
|
||||
ja := a.AsJulian()
|
||||
jb := b.AsJulian()
|
||||
if ja < jb {
|
||||
return -1
|
||||
}
|
||||
if ja > jb {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Logical
|
||||
if a.IsLogical() && b.IsLogical() {
|
||||
ba := a.AsBool()
|
||||
bb := b.AsBool()
|
||||
if ba == bb {
|
||||
return 0
|
||||
}
|
||||
if !ba {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// SqlGroupBy(aRows, aGroupColIdx, aAggSpecs) → aResult
|
||||
//
|
||||
// Go-native GROUP BY. Builds groups by hashing group-key columns,
|
||||
// then computes aggregates (SUM/AVG/COUNT/MIN/MAX) per group.
|
||||
//
|
||||
// aGroupColIdx: array of 1-based column indices for group key
|
||||
// aAggSpecs: array of {nColIdx, cFunc, nArgColIdx}
|
||||
// nColIdx: 1-based output position
|
||||
// cFunc: "SUM"/"AVG"/"COUNT"/"MIN"/"MAX"
|
||||
// nArgColIdx: 1-based column index of the argument (0 for COUNT(*))
|
||||
func SqlGroupBy(t *hbrt.Thread) {
|
||||
t.Frame(3, 0)
|
||||
defer t.EndProc()
|
||||
|
||||
rowsVal := t.Local(1)
|
||||
groupColsVal := t.Local(2)
|
||||
aggSpecsVal := t.Local(3)
|
||||
|
||||
if !rowsVal.IsArray() || !groupColsVal.IsArray() || !aggSpecsVal.IsArray() {
|
||||
t.PushValue(hbrt.MakeArray(0))
|
||||
t.RetValue()
|
||||
return
|
||||
}
|
||||
|
||||
rows := rowsVal.AsArray().Items
|
||||
nRows := len(rows)
|
||||
|
||||
// Parse group column indices
|
||||
gcArr := groupColsVal.AsArray().Items
|
||||
groupCols := make([]int, len(gcArr))
|
||||
for i, v := range gcArr {
|
||||
groupCols[i] = int(v.AsNumInt()) - 1
|
||||
}
|
||||
|
||||
// Parse aggregate specs
|
||||
type aggSpec struct {
|
||||
outCol int
|
||||
fn string
|
||||
argCol int
|
||||
}
|
||||
asArr := aggSpecsVal.AsArray().Items
|
||||
aggs := make([]aggSpec, len(asArr))
|
||||
for i, s := range asArr {
|
||||
arr := s.AsArray()
|
||||
if arr == nil || len(arr.Items) < 3 {
|
||||
continue
|
||||
}
|
||||
aggs[i] = aggSpec{
|
||||
outCol: int(arr.Items[0].AsNumInt()) - 1,
|
||||
fn: arr.Items[1].AsString(),
|
||||
argCol: int(arr.Items[2].AsNumInt()) - 1,
|
||||
}
|
||||
}
|
||||
|
||||
// Build groups
|
||||
type groupData struct {
|
||||
firstRow int
|
||||
count int
|
||||
sums []float64
|
||||
mins []hbrt.Value
|
||||
maxs []hbrt.Value
|
||||
counts []int
|
||||
}
|
||||
|
||||
groups := make(map[string]*groupData)
|
||||
groupOrder := make([]string, 0, 256)
|
||||
|
||||
nAggs := len(aggs)
|
||||
|
||||
for i := 0; i < nRows; i++ {
|
||||
ra := rows[i].AsArray()
|
||||
if ra == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Build group key
|
||||
key := ""
|
||||
for _, gc := range groupCols {
|
||||
if gc >= 0 && gc < len(ra.Items) {
|
||||
key += valueHashKey(ra.Items[gc]) + "|"
|
||||
}
|
||||
}
|
||||
|
||||
gd, exists := groups[key]
|
||||
if !exists {
|
||||
gd = &groupData{
|
||||
firstRow: i,
|
||||
sums: make([]float64, nAggs),
|
||||
mins: make([]hbrt.Value, nAggs),
|
||||
maxs: make([]hbrt.Value, nAggs),
|
||||
counts: make([]int, nAggs),
|
||||
}
|
||||
groups[key] = gd
|
||||
groupOrder = append(groupOrder, key)
|
||||
}
|
||||
gd.count++
|
||||
|
||||
// Accumulate aggregates
|
||||
for ai, ag := range aggs {
|
||||
if ag.fn == "COUNT" && ag.argCol < 0 {
|
||||
gd.counts[ai]++
|
||||
continue
|
||||
}
|
||||
if ag.argCol >= 0 && ag.argCol < len(ra.Items) {
|
||||
v := ra.Items[ag.argCol]
|
||||
if !v.IsNil() {
|
||||
gd.counts[ai]++
|
||||
if v.IsNumeric() {
|
||||
gd.sums[ai] += v.AsNumDouble()
|
||||
}
|
||||
if gd.mins[ai].IsNil() || compareValues(v, gd.mins[ai]) < 0 {
|
||||
gd.mins[ai] = v
|
||||
}
|
||||
if gd.maxs[ai].IsNil() || compareValues(v, gd.maxs[ai]) > 0 {
|
||||
gd.maxs[ai] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build result rows
|
||||
nCols := 0
|
||||
if nRows > 0 && rows[0].AsArray() != nil {
|
||||
nCols = len(rows[0].AsArray().Items)
|
||||
}
|
||||
|
||||
result := make([]hbrt.Value, 0, len(groups))
|
||||
for _, key := range groupOrder {
|
||||
gd := groups[key]
|
||||
srcRow := rows[gd.firstRow].AsArray()
|
||||
if srcRow == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Copy group columns from first row
|
||||
outItems := make([]hbrt.Value, nCols)
|
||||
copy(outItems, srcRow.Items)
|
||||
|
||||
// Fill aggregate values
|
||||
for ai, ag := range aggs {
|
||||
if ag.outCol >= 0 && ag.outCol < nCols {
|
||||
switch ag.fn {
|
||||
case "COUNT":
|
||||
outItems[ag.outCol] = hbrt.MakeNumInt(int64(gd.counts[ai]))
|
||||
case "SUM":
|
||||
if gd.counts[ai] > 0 {
|
||||
outItems[ag.outCol] = hbrt.MakeDoubleAuto(gd.sums[ai])
|
||||
} else {
|
||||
outItems[ag.outCol] = hbrt.MakeNil()
|
||||
}
|
||||
case "AVG":
|
||||
if gd.counts[ai] > 0 {
|
||||
outItems[ag.outCol] = hbrt.MakeDoubleAuto(gd.sums[ai] / float64(gd.counts[ai]))
|
||||
} else {
|
||||
outItems[ag.outCol] = hbrt.MakeNil()
|
||||
}
|
||||
case "MIN":
|
||||
outItems[ag.outCol] = gd.mins[ai]
|
||||
case "MAX":
|
||||
outItems[ag.outCol] = gd.maxs[ai]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result = append(result, hbrt.MakeArrayFrom(outItems))
|
||||
}
|
||||
|
||||
t.PushValue(hbrt.MakeArrayFrom(result))
|
||||
t.RetValue()
|
||||
}
|
||||
|
||||
// SqlEach(aFieldPositions, pcWhere, bBlock) → NIL
|
||||
//
|
||||
// Streaming variant of SqlScan — instead of materializing all matching
|
||||
|
||||
Reference in New Issue
Block a user