five/hbrdd/dbf/indexer.go

// Copyright (c) 2026 Charles KWON OhJun (charleskwonohjun@gmail.com)
// All rights reserved.

// DBFArea Indexer integration — connects NTX/CDX index engines to DBFArea.
// Implements hbrdd.Indexer interface on DBFArea.

package dbf

import (
	"bytes"
	"five/hbrt"
	"five/hbrdd"
	"five/hbrdd/cdx"
	"five/hbrdd/ntx"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

// IndexEngine is the common interface for NTX Index and CDX Tag.
type IndexEngine interface {
	Seek(searchKey []byte) (uint32, bool)
	GoTop() bool
	GoBottom() bool
	SkipNext() bool
	SkipPrev() bool
	CurRecNo() uint32
	CurKey() []byte
	IsEOF() bool
	IsBOF() bool
	KeyLen() int
	Close() error
}

// IndexWriter is optional: engines that support online maintenance
// implement this so APPEND / REPLACE / DELETE can patch the index
// in place. Engines that don't implement it become stale on any
// write — callers must REINDEX manually. NTX implements via
// rebuild (correct but O(N)); CDX online maintenance is not yet
// wired up so it falls back to the stale-index path for now.
type IndexWriter interface {
	InsertKey(key []byte, recNo uint32) error
	DeleteKey(recNo uint32) error
}

// indexState holds active index state for a DBFArea.
type indexState struct {
	indexes   []IndexEngine  // open NTX/CDX index engines
	names     []string       // index file paths
	tags      []string       // tag names (for display)
	current   int            // active index (-1 = natural order)
	keyExprs  []string       // key expressions for each index
	// Scope support
	scopeTop    []byte       // top scope key (nil = no scope)
	scopeBottom []byte       // bottom scope key (nil = no scope)
}

// KeyEvalFunc is a callback for evaluating index key expressions via the VM.
// Set by the generated code (via SetKeyEval) before calling OrderCreate.
// This allows evalKeyExprInner to call UDFs and evaluate complex expressions.
// Signature: func(exprString) → Value (called on the current Thread)
var KeyEvalFunc func(expr string) hbrt.Value

// keyRecordAsc/Desc implement sort.Interface for ntx.KeyRecord slices.
// Using concrete types (not sort.Slice with closure) avoids reflection and
// gives ~2x speedup on large index builds. Harbour: C qsort equivalent.
type keyRecordAsc []ntx.KeyRecord

func (ks keyRecordAsc) Len() int      { return len(ks) }
func (ks keyRecordAsc) Swap(i, j int) { ks[i], ks[j] = ks[j], ks[i] }
func (ks keyRecordAsc) Less(i, j int) bool {
	cmp := bytes.Compare(ks[i].Key, ks[j].Key)
	if cmp == 0 {
		return ks[i].RecNo < ks[j].RecNo
	}
	return cmp < 0
}

type keyRecordDesc []ntx.KeyRecord

func (ks keyRecordDesc) Len() int      { return len(ks) }
func (ks keyRecordDesc) Swap(i, j int) { ks[i], ks[j] = ks[j], ks[i] }
func (ks keyRecordDesc) Less(i, j int) bool {
	cmp := bytes.Compare(ks[i].Key, ks[j].Key)
	if cmp == 0 {
		return ks[i].RecNo < ks[j].RecNo
	}
	return cmp > 0
}

// ensureIndexState initializes the index state if nil.
func (a *DBFArea) ensureIndexState() {
	if a.idxState == nil {
		a.idxState = &indexState{current: -1}
	}
}

// OrderCreate creates a new index file. Equivalent to INDEX ON.
func (a *DBFArea) OrderCreate(params hbrdd.OrderCreateParams) error {
	a.ensureIndexState()

	// Flush pending record + update header/EOF before index build
	if a.dirty {
		a.flushRecord()
	}
	a.dataFile.WriteAt([]byte{EOFMarker}, a.header.EOFOffset())
	a.updateHeader()

	// Disable indexed navigation during key evaluation (GoTo must use natural order)
	a.idxState.current = -1

	idxPath := params.FilePath
	if idxPath == "" {
		return fmt.Errorf("index file path required")
	}

	// Determine index format: CDX if TAG specified or .cdx extension, otherwise NTX
	useCDX := params.TagName != "" || strings.HasSuffix(strings.ToLower(idxPath), ".cdx")
	if !strings.Contains(filepath.Base(idxPath), ".") {
		if useCDX {
			idxPath += ".cdx"
		} else {
			idxPath += ".ntx"
		}
	}

	// Build key evaluator from expression
	keyExpr := strings.ToUpper(params.KeyExpr)

	// Determine key length from first record (or default)
	keyLen := 10
	recCount, _ := a.RecCount()
	if recCount > 0 {
		sample := a.evalKeyExpr(keyExpr, 1)
		if len(sample) > 0 {
			keyLen = len(sample)
		}
	}

	// Build key records — apply FOR condition if present
	forExpr := strings.TrimSpace(params.ForExpr)
	keys := make([]ntx.KeyRecord, 0, recCount)

	// Fast path: pre-resolve simple field references for direct byte extraction.
	// Avoids per-record expression parsing, GoTo round-trips, and Value allocation.
	fieldSlices := a.resolveFieldSlices(keyExpr)

	if fieldSlices != nil && forExpr == "" {
		// Direct field byte extraction — zero Value allocation, sequential I/O
		recLen := int(a.header.RecordLen)
		headerLen := int(a.header.HeaderLen)
		// Pre-allocate a slab for all keys (single allocation)
		slab := make([]byte, int(recCount)*keyLen)
		for r := uint32(1); r <= recCount; r++ {
			k := slab[(r-1)*uint32(keyLen) : r*uint32(keyLen)]
			// Read record bytes (mmap or file)
			var rec []byte
			offset := int64(headerLen) + int64(r-1)*int64(recLen)
			if a.mmapData != nil && int(offset)+recLen <= len(a.mmapData) {
				rec = a.mmapData[offset : offset+int64(recLen)]
			} else {
				a.GoTo(r)
				a.loadRecord()
				rec = a.recBuf
			}
			// Copy field bytes directly into key, applying transforms inline.
			pos := 0
			for _, fs := range fieldSlices {
				end := pos + fs.len
				if end > keyLen {
					end = keyLen
				}
				n := end - pos
				if n > 0 {
					src := rec[fs.off : fs.off+n]
					switch {
					case fs.toUpper:
						for bi := 0; bi < n; bi++ {
							c := src[bi]
							if c >= 'a' && c <= 'z' {
								c -= 32
							}
							k[pos+bi] = c
						}
					case fs.toLower:
						for bi := 0; bi < n; bi++ {
							c := src[bi]
							if c >= 'A' && c <= 'Z' {
								c += 32
							}
							k[pos+bi] = c
						}
					default:
						copy(k[pos:end], src)
					}
				}
				pos = end
				if pos >= keyLen {
					break
				}
			}
			// Pad remainder with spaces
			for pos < keyLen {
				k[pos] = ' '
				pos++
			}
			keys = append(keys, ntx.KeyRecord{Key: k, RecNo: r})
		}
	} else if params.KeyFunc != nil {
		// Compiled path: gengo emitted an inline Go closure that evaluates
		// the key expression directly (no MacroEval string parsing).
		// ~3x faster than the MacroEval slow path for UDF indexes.
		// ForFunc — when also set by gengo — skips the runtime parser
		// for the FOR condition in the same way.
		slab := make([]byte, int(recCount)*keyLen)
		next := 0
		oldRec := a.recNo
		trimmedFor := strings.TrimSpace(forExpr)
		hasFor := trimmedFor != "" || params.ForFunc != nil
		for r := uint32(1); r <= recCount; r++ {
			a.GoTo(r)
			if hasFor {
				var include bool
				if params.ForFunc != nil {
					include = params.ForFunc()
				} else {
					include = a.evalForInner(trimmedFor)
				}
				if !include {
					continue
				}
			}
			val := params.KeyFunc()
			var src []byte
			if val.IsString() {
				src = []byte(val.AsString())
			} else if val.IsDate() {
				src = []byte(fmt.Sprintf("%08d", val.AsJulian()))
			} else {
				src = []byte(val.String())
			}
			k := slab[next : next+keyLen]
			next += keyLen
			n := copy(k, src)
			for j := n; j < keyLen; j++ {
				k[j] = ' '
			}
			keys = append(keys, ntx.KeyRecord{Key: k, RecNo: r})
		}
		a.GoTo(oldRec)
	} else {
		// MacroEval slow path: string-based expression evaluation.
		// Used only when gengo can't emit a compiled closure (rare edge cases).
		slab := make([]byte, int(recCount)*keyLen)
		next := 0
		oldRec := a.recNo
		trimmedKey := strings.TrimSpace(keyExpr)
		trimmedFor := strings.TrimSpace(forExpr)
		hasFor := trimmedFor != "" || params.ForFunc != nil
		for r := uint32(1); r <= recCount; r++ {
			a.GoTo(r)
			if hasFor {
				var include bool
				if params.ForFunc != nil {
					include = params.ForFunc()
				} else {
					include = a.evalForInner(trimmedFor)
				}
				if !include {
					continue
				}
			}
			src := a.evalKeyExprInner(trimmedKey)
			k := slab[next : next+keyLen]
			next += keyLen
			n := copy(k, src)
			for j := n; j < keyLen; j++ {
				k[j] = ' '
			}
			keys = append(keys, ntx.KeyRecord{Key: k, RecNo: r})
		}
		a.GoTo(oldRec)
	}

	// Sort keys before building index.
	// Harbour: equal keys ordered by RecNo ascending (stable by record number).
	// Use concrete sort.Interface (no reflection) + branch hoist for ~2x speedup
	// over sort.Slice with closure.
	if params.Descending {
		sort.Sort(keyRecordDesc(keys))
	} else {
		sort.Sort(keyRecordAsc(keys))
	}

	if useCDX {
		// CDX compound index — append tag to existing file or create new
		tagName := params.TagName
		if tagName == "" {
			tagName = keyExpr // default tag name = key expression
		}
		ci, err := cdx.CreateOrAddTag(idxPath, tagName, keyExpr, params.ForExpr,
			keyLen, params.Unique, params.Descending, keys)
		if err != nil {
			return fmt.Errorf("create CDX index failed: %w", err)
		}
		// Register all tags from the CDX file
		// If this is the first tag, add all; if adding to existing, re-register
		// Remove old entries for this CDX file first
		newIndexes := make([]IndexEngine, 0, len(a.idxState.indexes)+ci.TagCount())
		newNames := make([]string, 0, cap(newIndexes))
		newTags := make([]string, 0, cap(newIndexes))
		newKeyExprs := make([]string, 0, cap(newIndexes))
		for i, name := range a.idxState.names {
			if name != idxPath {
				newIndexes = append(newIndexes, a.idxState.indexes[i])
				newNames = append(newNames, a.idxState.names[i])
				newTags = append(newTags, a.idxState.tags[i])
				newKeyExprs = append(newKeyExprs, a.idxState.keyExprs[i])
			}
		}
		for _, tag := range ci.Tags() {
			newIndexes = append(newIndexes, tag)
			newNames = append(newNames, idxPath)
			newTags = append(newTags, tag.Name)
			newKeyExprs = append(newKeyExprs, tag.KeyExpr())
		}
		a.idxState.indexes = newIndexes
		a.idxState.names = newNames
		a.idxState.tags = newTags
		a.idxState.keyExprs = newKeyExprs
	} else {
		idx, err := ntx.CreateIndex(idxPath, keyExpr, keyLen, params.Unique, params.Descending, keys)
		if err != nil {
			return fmt.Errorf("create index failed: %w", err)
		}
		a.idxState.indexes = append(a.idxState.indexes, idx)
		a.idxState.names = append(a.idxState.names, idxPath)
		a.idxState.tags = append(a.idxState.tags, params.TagName)
		a.idxState.keyExprs = append(a.idxState.keyExprs, keyExpr)
	}
	a.idxState.current = len(a.idxState.indexes) - 1

	return nil
}

// OrderListAdd opens an existing index file (NTX single-order or CDX compound).
func (a *DBFArea) OrderListAdd(path string) error {
	a.ensureIndexState()

	// Auto-detect extension: try .cdx first, then .ntx
	if !strings.Contains(filepath.Base(path), ".") {
		if _, err := os.Stat(path + ".cdx"); err == nil {
			path += ".cdx"
		} else {
			path += ".ntx"
		}
	}

	ext := strings.ToLower(filepath.Ext(path))

	if ext == ".cdx" {
		// CDX compound index — opens all tags
		ci, err := cdx.OpenIndex(path)
		if err != nil {
			return fmt.Errorf("open CDX failed: %w", err)
		}
		for _, tag := range ci.Tags() {
			a.idxState.indexes = append(a.idxState.indexes, tag)
			a.idxState.names = append(a.idxState.names, path)
			a.idxState.tags = append(a.idxState.tags, tag.Name)
			a.idxState.keyExprs = append(a.idxState.keyExprs, tag.KeyExpr())
		}
		if len(ci.Tags()) > 0 {
			a.idxState.current = len(a.idxState.indexes) - len(ci.Tags()) // first tag
		}
		return nil
	}

	// NTX single index
	idx, err := ntx.OpenIndex(path)
	if err != nil {
		return fmt.Errorf("open index failed: %w", err)
	}

	a.idxState.indexes = append(a.idxState.indexes, idx)
	a.idxState.names = append(a.idxState.names, path)
	a.idxState.tags = append(a.idxState.tags, "")
	/* Pull the key expression out of the on-disk NTX header so DBOI_EXPRESSION
	 * works after re-opening an index file. Previously we appended "" here,
	 * which silently broke MatchOrderByTag (TSqlIndex.prg) — the substring
	 * test against an empty string always failed, so SELECT … ORDER BY <col>
	 * LIMIT N could never recognize an existing tag and skipped the LIMIT
	 * pushdown / sort-skip optimizations. */
	a.idxState.keyExprs = append(a.idxState.keyExprs, idx.KeyExpr())
	a.idxState.current = len(a.idxState.indexes) - 1

	return nil
}

// OrderListClear closes all index files.
func (a *DBFArea) OrderListClear() error {
	if a.idxState == nil {
		return nil
	}
	for _, idx := range a.idxState.indexes {
		idx.Close()
	}
	a.idxState = &indexState{current: -1}
	return nil
}

// OrderListFocus sets the active index by tag name, number, or file name.
// Harbour: OrdSetFocus(nOrder) or OrdSetFocus("tagName")
func (a *DBFArea) OrderListFocus(tagName string) error {
	a.ensureIndexState()
	if tagName == "" || tagName == "0" {
		a.idxState.current = -1 // natural order
		a.ClearScope()
		return nil
	}

	// Try as numeric order (1-based)
	if n, err := parseOrderNum(tagName); err == nil {
		if n == 0 {
			a.idxState.current = -1
			a.ClearScope()
			return nil
		}
		if n >= 1 && n <= len(a.idxState.indexes) {
			a.idxState.current = n - 1
			a.ClearScope()
			return nil
		}
	}

	upper := strings.ToUpper(tagName)
	// Match by tag name
	for i, name := range a.idxState.tags {
		if strings.ToUpper(name) == upper {
			a.idxState.current = i
			a.ClearScope()
			return nil
		}
	}
	// Match by file name
	for i, name := range a.idxState.names {
		base := strings.ToUpper(filepath.Base(name))
		ext := strings.ToUpper(filepath.Ext(name))
		if base == upper || strings.TrimSuffix(base, ext) == upper {
			a.idxState.current = i
			a.ClearScope()
			return nil
		}
	}
	return fmt.Errorf("index not found: %s", tagName)
}

// parseOrderNum tries to parse a string as a positive integer (order number).
func parseOrderNum(s string) (int, error) {
	s = strings.TrimSpace(s)
	if len(s) == 0 {
		return 0, fmt.Errorf("empty")
	}
	n := 0
	for _, c := range s {
		if c < '0' || c > '9' {
			return 0, fmt.Errorf("not a number")
		}
		n = n*10 + int(c-'0')
	}
	return n, nil
}

// OrderListRebuild rebuilds all indexes.
// Harbour: ORDLISTREBUILD / REINDEX — recreates all open indexes from current data.
func (a *DBFArea) OrderListRebuild() error {
	if a.idxState == nil || len(a.idxState.indexes) == 0 {
		return nil
	}

	// Save current index info
	savedCurrent := a.idxState.current
	type idxInfo struct {
		name    string
		tag     string
		keyExpr string
	}
	infos := make([]idxInfo, len(a.idxState.indexes))
	for i := range a.idxState.indexes {
		infos[i] = idxInfo{
			name:    a.idxState.names[i],
			tag:     a.idxState.tags[i],
			keyExpr: a.idxState.keyExprs[i],
		}
	}

	// Close all indexes and disable indexed navigation
	for _, idx := range a.idxState.indexes {
		idx.Close()
	}
	a.idxState.indexes = nil
	a.idxState.names = nil
	a.idxState.tags = nil
	a.idxState.keyExprs = nil
	a.idxState.current = -1

	// Remove idxState so GoTo uses natural order during rebuild
	a.idxState = nil

	// Recreate each index
	for _, info := range infos {
		err := a.OrderCreate(hbrdd.OrderCreateParams{
			KeyExpr:  info.keyExpr,
			FilePath: info.name,
			TagName:  info.tag,
		})
		if err != nil {
			return fmt.Errorf("rebuild index %s: %w", info.name, err)
		}
	}

	// Restore active index
	if a.idxState != nil && savedCurrent >= 0 && savedCurrent < len(a.idxState.indexes) {
		a.idxState.current = savedCurrent
	}

	return nil
}

// OrderDestroy removes an index file.
func (a *DBFArea) OrderDestroy(tagName string) error {
	a.ensureIndexState()
	upper := strings.ToUpper(tagName)
	for i, name := range a.idxState.tags {
		if strings.ToUpper(name) == upper {
			a.idxState.indexes[i].Close()
			os.Remove(a.idxState.names[i])
			// Remove from slices
			a.idxState.indexes = append(a.idxState.indexes[:i], a.idxState.indexes[i+1:]...)
			a.idxState.names = append(a.idxState.names[:i], a.idxState.names[i+1:]...)
			a.idxState.tags = append(a.idxState.tags[:i], a.idxState.tags[i+1:]...)
			a.idxState.keyExprs = append(a.idxState.keyExprs[:i], a.idxState.keyExprs[i+1:]...)
			if a.idxState.current >= len(a.idxState.indexes) {
				a.idxState.current = -1
			}
			return nil
		}
	}
	return fmt.Errorf("index not found: %s", tagName)
}

// OrderInfo returns information about an index order.
func (a *DBFArea) OrderInfo(ordNo int) (*hbrdd.OrderInfo, error) {
	a.ensureIndexState()
	idx := ordNo - 1
	if idx < 0 || idx >= len(a.idxState.indexes) {
		return nil, fmt.Errorf("invalid order number: %d", ordNo)
	}
	return &hbrdd.OrderInfo{
		Name:    a.idxState.tags[idx],
		KeyExpr: a.idxState.keyExprs[idx],
	}, nil
}

// Seek searches for a key in the active index.
// Harbour compatible: partial key matching, softseek, space padding.
func (a *DBFArea) Seek(key hbrt.Value, softSeek bool, findLast bool) (bool, error) {
	a.ensureIndexState()
	if a.idxState.current < 0 || a.idxState.current >= len(a.idxState.indexes) {
		return false, fmt.Errorf("no active index")
	}

	idx := a.idxState.indexes[a.idxState.current]
	keyLen := idx.KeyLen()

	// Convert key to bytes and track actual search length
	var searchKey []byte
	var actualLen int

	if key.IsString() {
		s := key.AsString()
		actualLen = len(s)
		// Pad with spaces to full key length (Harbour convention)
		if actualLen < keyLen {
			padded := make([]byte, keyLen)
			copy(padded, []byte(s))
			for i := actualLen; i < keyLen; i++ {
				padded[i] = ' '
			}
			searchKey = padded
		} else {
			searchKey = []byte(s[:keyLen])
			actualLen = keyLen
		}
	} else if key.IsNumeric() {
		s := fmt.Sprintf("%*d", keyLen, key.AsNumInt())
		searchKey = []byte(s)
		if len(searchKey) > keyLen {
			searchKey = searchKey[:keyLen]
		}
		actualLen = keyLen
	} else {
		searchKey = []byte(key.AsString())
		actualLen = len(searchKey)
	}

	// Seek in index
	recNo, exactFound := idx.Seek(searchKey)

	// If not exact, check partial match: compare only actualLen bytes
	if !exactFound && recNo > 0 && actualLen < keyLen {
		// Position at the found location and check partial match
		curKey := idx.CurKey()
		if len(curKey) >= actualLen && bytes.Equal(curKey[:actualLen], searchKey[:actualLen]) {
			exactFound = true
		}
	}

	if exactFound && recNo > 0 {
		a.GoTo(recNo)
		a.FEof = false
		// SET DELETED ON: if found record is deleted, skip to next non-deleted with same key
		if hbrdd.IsSetDeleted != nil && hbrdd.IsSetDeleted() && a.Deleted() {
			// Skip forward through deleted records
			for {
				idx.SkipNext()
				if idx.IsEOF() {
					break
				}
				// Check if key still matches (partial or full)
				curKey := idx.CurKey()
				if actualLen < keyLen {
					if !bytes.Equal(curKey[:actualLen], searchKey[:actualLen]) {
						break
					}
				} else {
					if !bytes.Equal(curKey, searchKey) {
						break
					}
				}
				a.GoTo(idx.CurRecNo())
				if !a.Deleted() {
					a.SetFound(true)
					return true, nil
				}
			}
			// All matching records are deleted
			rc, _ := a.RecCount()
			a.GoTo(rc + 1)
			a.FEof = true
			a.SetFound(false)
			return false, nil
		}
		a.SetFound(true)
		return true, nil
	}

	if softSeek && !idx.IsEOF() {
		// Softseek: position at the next higher key
		posRecNo := idx.CurRecNo()
		if posRecNo > 0 {
			a.GoTo(posRecNo)
			a.FEof = false
			a.SetFound(false)
			return false, nil
		}
	}

	// Not found — go to EOF
	rc, _ := a.RecCount()
	a.GoTo(rc + 1)
	a.FEof = true
	a.SetFound(false)
	return false, nil
}

// GoTopIndexed positions at the first key in the active index.
// Harbour: if SCOPE is set, positions at the first key >= scopeTop.
func (a *DBFArea) GoTopIndexed() error {
	if a.idxState == nil || a.idxState.current < 0 {
		return a.GoTop()
	}
	idx := a.idxState.indexes[a.idxState.current]

	if a.idxState.scopeTop != nil {
		// Seek to scope top boundary
		recNo, _ := idx.Seek(a.idxState.scopeTop)
		if recNo == 0 || idx.IsEOF() {
			rc, _ := a.RecCount()
			a.FEof = true
			return a.GoTo(rc + 1)
		}
		// Check if within bottom scope
		if a.idxState.scopeBottom != nil {
			if bytes.Compare(idx.CurKey(), a.idxState.scopeBottom) > 0 {
				rc, _ := a.RecCount()
				a.FEof = true
				return a.GoTo(rc + 1)
			}
		}
		return a.GoTo(idx.CurRecNo())
	}

	idx.GoTop()
	if idx.IsEOF() {
		rc, _ := a.RecCount()
		a.FEof = true
		return a.GoTo(rc + 1)
	}
	a.GoTo(idx.CurRecNo())
	// Skip deleted records at top
	if hbrdd.IsSetDeleted != nil && hbrdd.IsSetDeleted() && a.Deleted() {
		return a.SkipIndexed(1)
	}
	return nil
}

// GoBottomIndexed positions at the last key in the active index.
// Harbour: if SCOPE is set, positions at the last key <= scopeBottom.
func (a *DBFArea) GoBottomIndexed() error {
	if a.idxState == nil || a.idxState.current < 0 {
		return a.GoBottom()
	}
	idx := a.idxState.indexes[a.idxState.current]

	if a.idxState.scopeBottom != nil {
		// Seek to scope bottom boundary
		_, exact := idx.Seek(a.idxState.scopeBottom)
		if idx.IsEOF() {
			// All keys less than bottom scope — go to physical bottom
			idx.GoBottom()
		} else if !exact {
			// Positioned past bottom — go back one
			idx.SkipPrev()
		} else {
			// Exact match — skip forward to last matching key, then position there
			for {
				idx.SkipNext()
				if idx.IsEOF() || bytes.Compare(idx.CurKey(), a.idxState.scopeBottom) > 0 {
					idx.SkipPrev()
					break
				}
			}
		}
		if idx.IsBOF() || idx.IsEOF() {
			a.FBof = true
			return a.GoTo(1)
		}
		// Verify within top scope
		if a.idxState.scopeTop != nil {
			if bytes.Compare(idx.CurKey(), a.idxState.scopeTop) < 0 {
				a.FEof = true
				rc, _ := a.RecCount()
				return a.GoTo(rc + 1)
			}
		}
		return a.GoTo(idx.CurRecNo())
	}

	idx.GoBottom()
	if idx.IsBOF() {
		return a.GoTo(1)
	}
	return a.GoTo(idx.CurRecNo())
}

// SkipIndexed skips using the active index order.
// Harbour: respects SCOPE boundaries — stops at scope edges.
func (a *DBFArea) SkipIndexed(count int64) error {
	if a.idxState == nil || a.idxState.current < 0 {
		return a.Skip(count)
	}
	idx := a.idxState.indexes[a.idxState.current]
	hasScope := a.idxState.scopeTop != nil || a.idxState.scopeBottom != nil

	setDel := hbrdd.IsSetDeleted != nil && hbrdd.IsSetDeleted()

	if count > 0 {
		for i := int64(0); i < count; i++ {
			for {
				idx.SkipNext()
				if idx.IsEOF() || idx.CurRecNo() == 0 {
					rc, _ := a.RecCount()
					a.GoTo(rc + 1)
					a.FEof = true
					return nil
				}
				// Check bottom scope
				if hasScope && a.idxState.scopeBottom != nil {
					if bytes.Compare(idx.CurKey(), a.idxState.scopeBottom) > 0 {
						rc, _ := a.RecCount()
						a.GoTo(rc + 1)
						a.FEof = true
						return nil
					}
				}
				// Skip deleted records
				if setDel {
					a.GoTo(idx.CurRecNo())
					if a.Deleted() {
						continue
					}
				}
				break
			}
		}
	} else if count < 0 {
		for i := int64(0); i > count; i-- {
			idx.SkipPrev()
			if idx.IsBOF() {
				// Stay at first record in scope
				if a.idxState.scopeTop != nil {
					idx.Seek(a.idxState.scopeTop)
				} else {
					idx.GoTop()
				}
				if !idx.IsEOF() {
					a.GoTo(idx.CurRecNo())
				} else {
					a.GoTo(1)
				}
				a.FBof = true // set AFTER GoTo (GoTo resets FBof)
				return nil
			}
			// Check top scope
			if hasScope && a.idxState.scopeTop != nil {
				if bytes.Compare(idx.CurKey(), a.idxState.scopeTop) < 0 {
					a.FBof = true
					idx.Seek(a.idxState.scopeTop)
					if !idx.IsEOF() {
						return a.GoTo(idx.CurRecNo())
					}
					return a.GoTo(1)
				}
			}
		}
	}
	return a.GoTo(idx.CurRecNo())
}

// --- Scope support (ORDSCOPE) ---

// SetScope sets top and/or bottom scope boundaries for the active index.
// Harbour: OrdScope(TOPSCOPE, val) / OrdScope(BOTTOMSCOPE, val)
// Pass zero-value hbrt.Value{} (not MakeNil) to skip setting that boundary.
func (a *DBFArea) SetScope(top, bottom hbrt.Value) error {
	a.ensureIndexState()
	if a.idxState.current < 0 {
		return fmt.Errorf("no active index")
	}
	idx := a.idxState.indexes[a.idxState.current]
	keyLen := idx.KeyLen()

	if !top.IsNil() && top.Type() != 0 {
		a.idxState.scopeTop = scopeKeyFromValue(top, keyLen)
	}
	if !bottom.IsNil() && bottom.Type() != 0 {
		a.idxState.scopeBottom = scopeKeyFromValue(bottom, keyLen)
	}
	return nil
}

// SetScopeTop sets only the top scope.
func (a *DBFArea) SetScopeTop(val hbrt.Value) {
	a.ensureIndexState()
	if a.idxState.current < 0 {
		return
	}
	keyLen := a.idxState.indexes[a.idxState.current].KeyLen()
	a.idxState.scopeTop = scopeKeyFromValue(val, keyLen)
}

// SetScopeBottom sets only the bottom scope.
func (a *DBFArea) SetScopeBottom(val hbrt.Value) {
	a.ensureIndexState()
	if a.idxState.current < 0 {
		return
	}
	keyLen := a.idxState.indexes[a.idxState.current].KeyLen()
	a.idxState.scopeBottom = scopeKeyFromValue(val, keyLen)
}

// ClearScope removes all scope boundaries.
func (a *DBFArea) ClearScope() error {
	if a.idxState != nil {
		a.idxState.scopeTop = nil
		a.idxState.scopeBottom = nil
	}
	return nil
}

// ClearScopeTop removes only the top scope boundary.
func (a *DBFArea) ClearScopeTop() {
	if a.idxState != nil {
		a.idxState.scopeTop = nil
	}
}

// ClearScopeBottom removes only the bottom scope boundary.
func (a *DBFArea) ClearScopeBottom() {
	if a.idxState != nil {
		a.idxState.scopeBottom = nil
	}
}

// GetScopeTop returns the current top scope key (nil if none).
func (a *DBFArea) GetScopeTop() []byte {
	if a.idxState != nil {
		return a.idxState.scopeTop
	}
	return nil
}

// GetScopeBottom returns the current bottom scope key (nil if none).
func (a *DBFArea) GetScopeBottom() []byte {
	if a.idxState != nil {
		return a.idxState.scopeBottom
	}
	return nil
}

// scopeKeyFromValue converts a Harbour Value to a scope key byte slice.
func scopeKeyFromValue(v hbrt.Value, keyLen int) []byte {
	var key []byte
	if v.IsString() {
		key = []byte(v.AsString())
	} else if v.IsNumeric() {
		key = []byte(fmt.Sprintf("%*d", keyLen, v.AsNumInt()))
	} else {
		key = []byte(v.AsString())
	}
	// Pad to keyLen
	if len(key) < keyLen {
		padded := make([]byte, keyLen)
		copy(padded, key)
		for i := len(key); i < keyLen; i++ {
			padded[i] = ' '
		}
		return padded
	}
	if len(key) > keyLen {
		return key[:keyLen]
	}
	return key
}

// --- Index info accessors ---

// IndexCount returns the number of open indexes.
func (a *DBFArea) IndexCount() int {
	if a.idxState == nil {
		return 0
	}
	return len(a.idxState.indexes)
}

// CurrentOrder returns the 1-based current order number (0 = natural).
func (a *DBFArea) CurrentOrder() int {
	if a.idxState == nil || a.idxState.current < 0 {
		return 0
	}
	return a.idxState.current + 1
}

// OrderName returns the tag name for order n (1-based).
func (a *DBFArea) OrderName(n int) string {
	if a.idxState == nil || n < 1 || n > len(a.idxState.tags) {
		return ""
	}
	return a.idxState.tags[n-1]
}

// OrderKeyExpr returns the key expression for order n (1-based).
func (a *DBFArea) OrderKeyExpr(n int) string {
	if a.idxState == nil || n < 1 || n > len(a.idxState.keyExprs) {
		return ""
	}
	return a.idxState.keyExprs[n-1]
}

// OrderKeyLen returns the byte length of keys stored in order n (1-based).
// Zero means "unknown" (no such order, or indexes slice stale).
func (a *DBFArea) OrderKeyLen(n int) int {
	if a.idxState == nil || n < 1 || n > len(a.idxState.indexes) {
		return 0
	}
	return a.idxState.indexes[n-1].KeyLen()
}

// fieldSlice describes a direct byte range within a record buffer.
// The optional transform is applied during key extraction (e.g. UPPER/LOWER).
type fieldSlice struct {
	off       int  // byte offset in record (including deletion flag)
	len       int  // byte length
	toUpper   bool // apply ASCII UPPER during extraction
	toLower   bool // apply ASCII LOWER during extraction
	numeric   bool // DBF numeric field (space-padded left; copy as-is for ASCII compare)
}

// resolveFieldSlices attempts to resolve a key expression into direct record byte ranges.
// Returns nil if the expression contains things that require full evaluation.
// Supports:
//   - Simple field names (CHAR and Numeric)
//   - FIELD->NAME / _FIELD->NAME / alias->NAME
//   - "+" concatenation of the above
//   - UPPER(field), LOWER(field) — CHAR fields only
func (a *DBFArea) resolveFieldSlices(expr string) []fieldSlice {
	expr = strings.TrimSpace(expr)
	if expr == "" {
		return nil
	}

	// Split on "+" for concatenation (but only top-level, not inside function args)
	parts := splitTopLevel(expr, '+')

	var slices []fieldSlice
	for _, part := range parts {
		part = strings.TrimSpace(part)
		if part == "" {
			return nil
		}

		toUpper, toLower := false, false

		// UPPER( ... ) / LOWER( ... ) wrapper
		upperPart := strings.ToUpper(part)
		if strings.HasPrefix(upperPart, "UPPER(") && strings.HasSuffix(part, ")") {
			toUpper = true
			part = strings.TrimSpace(part[6 : len(part)-1])
			upperPart = strings.ToUpper(part)
		} else if strings.HasPrefix(upperPart, "LOWER(") && strings.HasSuffix(part, ")") {
			toLower = true
			part = strings.TrimSpace(part[6 : len(part)-1])
			upperPart = strings.ToUpper(part)
		}

		// Any remaining "(" means nested function — fall back to slow path
		if strings.Contains(part, "(") {
			return nil
		}

		// Strip FIELD-> / _FIELD-> / alias-> prefix
		fieldName := upperPart
		if idx := strings.Index(fieldName, "->"); idx >= 0 {
			fieldName = strings.TrimSpace(fieldName[idx+2:])
		}

		// Look up field
		found := false
		for i := 0; i < len(a.fieldDescs); i++ {
			fi := a.GetFieldInfo(i)
			if strings.ToUpper(fi.Name) == fieldName {
				ft := a.fieldDescs[i].Type
				isChar := ft == 'C' || ft == 'c'
				isNum := ft == 'N' || ft == 'n' || ft == 'F' || ft == 'f'
				// UPPER/LOWER requires CHAR
				if (toUpper || toLower) && !isChar {
					return nil
				}
				if !isChar && !isNum {
					return nil
				}
				slices = append(slices, fieldSlice{
					off:     int(a.offsets[i]),
					len:     int(a.fieldDescs[i].Len),
					toUpper: toUpper,
					toLower: toLower,
					numeric: isNum,
				})
				found = true
				break
			}
		}
		if !found {
			return nil
		}
	}
	return slices
}

// splitTopLevel splits expr on delimiter, but only at the top level (not inside parens).
func splitTopLevel(expr string, delim byte) []string {
	var parts []string
	depth := 0
	start := 0
	for i := 0; i < len(expr); i++ {
		switch expr[i] {
		case '(':
			depth++
		case ')':
			depth--
		case delim:
			if depth == 0 {
				parts = append(parts, expr[start:i])
				start = i + 1
			}
		}
	}
	parts = append(parts, expr[start:])
	return parts
}

// evalKeyExpr evaluates an index key expression for a given record.
// Supports: field names, UPPER(), LOWER(), LTRIM(), RTRIM(), ALLTRIM(),
// STR(), DTOS(), SUBSTR(), LEFT(), RIGHT(), PADL(), PADR(),
// field1+field2 (concatenation), nested functions.
func (a *DBFArea) evalKeyExpr(expr string, recNo uint32) []byte {
	oldRecNo := a.recNo
	a.GoTo(recNo)
	result := a.evalKeyExprInner(strings.TrimSpace(expr))
	a.GoTo(oldRecNo)
	return result
}

func (a *DBFArea) evalKeyExprInner(expr string) []byte {
	upper := strings.ToUpper(expr)

	// String literal
	if len(expr) >= 2 && expr[0] == '"' && expr[len(expr)-1] == '"' {
		return []byte(expr[1 : len(expr)-1])
	}

	// Strip FIELD-> or _FIELD-> or alias-> prefix (Harbour: M->var, FIELD->var)
	fieldName := strings.TrimSpace(upper)
	if idx := strings.Index(fieldName, "->"); idx >= 0 {
		fieldName = strings.TrimSpace(fieldName[idx+2:])
	}

	// Simple field name
	for i := 0; i < a.FieldCount(); i++ {
		fi := a.GetFieldInfo(i)
		if strings.ToUpper(fi.Name) == fieldName {
			val, _ := a.GetValue(i)
			return formatKeyValue(val, fi)
		}
	}

	// Function calls: FUNC(args)
	if parenOpen := strings.Index(expr, "("); parenOpen > 0 {
		funcName := strings.ToUpper(strings.TrimSpace(expr[:parenOpen]))
		// Find matching close paren
		parenClose := findMatchingParen(expr, parenOpen)
		if parenClose < 0 {
			parenClose = len(expr) - 1
		}
		argsStr := expr[parenOpen+1 : parenClose]

		switch funcName {
		case "UPPER":
			inner := a.evalKeyExprInner(argsStr)
			return []byte(strings.ToUpper(string(inner)))
		case "LOWER":
			inner := a.evalKeyExprInner(argsStr)
			return []byte(strings.ToLower(string(inner)))
		case "ALLTRIM", "TRIM":
			inner := a.evalKeyExprInner(argsStr)
			return []byte(strings.TrimSpace(string(inner)))
		case "LTRIM":
			inner := a.evalKeyExprInner(argsStr)
			return []byte(strings.TrimLeft(string(inner), " "))
		case "RTRIM":
			inner := a.evalKeyExprInner(argsStr)
			return []byte(strings.TrimRight(string(inner), " "))
		case "LEFT":
			args := splitArgs(argsStr)
			if len(args) >= 2 {
				inner := a.evalKeyExprInner(args[0])
				n := parseIntIdx(args[1])
				if n > len(inner) {
					n = len(inner)
				}
				return inner[:n]
			}
		case "RIGHT":
			args := splitArgs(argsStr)
			if len(args) >= 2 {
				inner := a.evalKeyExprInner(args[0])
				n := parseIntIdx(args[1])
				if n > len(inner) {
					n = len(inner)
				}
				return inner[len(inner)-n:]
			}
		case "SUBSTR":
			args := splitArgs(argsStr)
			if len(args) >= 2 {
				inner := a.evalKeyExprInner(args[0])
				start := parseIntIdx(args[1]) - 1 // 1-based to 0-based
				if start < 0 {
					start = 0
				}
				length := len(inner) - start
				if len(args) >= 3 {
					length = parseIntIdx(args[2])
				}
				if start+length > len(inner) {
					length = len(inner) - start
				}
				return inner[start : start+length]
			}
		case "STR":
			args := splitArgs(argsStr)
			inner := a.evalKeyExprInner(args[0])
			if len(args) >= 2 {
				width := parseIntIdx(args[1])
				s := string(inner)
				return []byte(fmt.Sprintf("%*s", width, strings.TrimSpace(s)))
			}
			return inner
		case "DTOS":
			inner := a.evalKeyExprInner(argsStr)
			// Date → YYYYMMDD sortable string
			return inner
		case "PADL":
			args := splitArgs(argsStr)
			if len(args) >= 2 {
				inner := string(a.evalKeyExprInner(args[0]))
				width := parseIntIdx(args[1])
				fill := " "
				if len(args) >= 3 {
					fill = strings.Trim(args[2], "\"' ")
					if fill == "" {
						fill = " "
					}
				}
				for len(inner) < width {
					inner = fill + inner
				}
				return []byte(inner[:width])
			}
		case "PADR":
			args := splitArgs(argsStr)
			if len(args) >= 2 {
				inner := string(a.evalKeyExprInner(args[0]))
				width := parseIntIdx(args[1])
				for len(inner) < width {
					inner = inner + " "
				}
				return []byte(inner[:width])
			}
		default:
			// Unknown function — use VM MacroEval for UDF calls
			if KeyEvalFunc != nil {
				fullExpr := expr[:parenOpen] + "(" + argsStr + ")"
				val := KeyEvalFunc(fullExpr)
				return valueToKeyBytes(val)
			}
			// Fallback: evaluate inner as field
			return a.evalKeyExprInner(argsStr)
		}
	}

	// Concatenation: expr1 + expr2 (find + not inside parens)
	if plus := findOperator(expr, '+'); plus > 0 {
		left := a.evalKeyExprInner(expr[:plus])
		right := a.evalKeyExprInner(expr[plus+1:])
		return append(left, right...)
	}

	// Numeric literal
	s := strings.TrimSpace(expr)
	if len(s) > 0 && (s[0] >= '0' && s[0] <= '9') {
		return []byte(s)
	}

	// Final fallback: use VM MacroEval for any unresolvable expression
	if KeyEvalFunc != nil {
		val := KeyEvalFunc(expr)
		return valueToKeyBytes(val)
	}

	return []byte(expr)
}

// evalForExpr evaluates a FOR condition for a given record. Returns true if record matches.
// Supports: FIELD = "value", FIELD = value, FIELD > value, !DELETED(), .T., .F.
func (a *DBFArea) evalForExpr(forExpr string, recNo uint32) bool {
	oldRecNo := a.recNo
	a.GoTo(recNo)
	result := a.evalForInner(strings.TrimSpace(forExpr))
	a.GoTo(oldRecNo)
	return result
}

func (a *DBFArea) evalForInner(expr string) bool {
	upper := strings.ToUpper(strings.TrimSpace(expr))

	if upper == ".T." || upper == "TRUE" {
		return true
	}
	if upper == ".F." || upper == "FALSE" {
		return false
	}
	if upper == "!DELETED()" || upper == ".NOT. DELETED()" {
		return !a.Deleted()
	}
	if upper == "DELETED()" {
		return a.Deleted()
	}

	// FIELD = "value" or FIELD = value
	for _, op := range []string{"==", "=", "!=", "<>", ">=", "<=", ">", "<"} {
		if idx := strings.Index(expr, op); idx > 0 {
			leftExpr := strings.TrimSpace(expr[:idx])
			rightExpr := strings.TrimSpace(expr[idx+len(op):])

			leftVal := string(a.evalKeyExprInner(leftExpr))
			rightVal := strings.Trim(rightExpr, "\"' ")

			leftTrim := strings.TrimRight(leftVal, " ")
			switch op {
			case "=", "==":
				return leftTrim == rightVal || leftVal == rightVal
			case "!=", "<>":
				return leftTrim != rightVal && leftVal != rightVal
			case ">":
				return leftTrim > rightVal
			case "<":
				return leftTrim < rightVal
			case ">=":
				return leftTrim >= rightVal
			case "<=":
				return leftTrim <= rightVal
			}
		}
	}

	// .AND. / .OR.
	if idx := strings.Index(upper, ".AND."); idx > 0 {
		left := a.evalForInner(expr[:idx])
		right := a.evalForInner(expr[idx+5:])
		return left && right
	}
	if idx := strings.Index(upper, ".OR."); idx > 0 {
		left := a.evalForInner(expr[:idx])
		right := a.evalForInner(expr[idx+4:])
		return left || right
	}

	return true // default: include record
}

// valueToKeyBytes converts a hbrt.Value to index key bytes.
//
// Numerics are encoded so that lexicographic byte order matches
// numeric order *including negatives*. The previous "%20.10f"
// format was wrong for any negative key: " 99" (space-padded) and
// "-100" both fit in 20 chars, but ' ' (0x20) sorts BEFORE '-'
// (0x2D), so -100 was indexed as GREATER than 99. Any NTX/CDX
// built over a column that ever held a negative number returned
// wrong rows for SEEK / range scans.
//
// Encoding: 1-byte sign prefix + 21-byte zero-padded magnitude.
//   * positive / zero  → '1' + "%021.10f" of value
//   * negative         → '0' + digitComplement("%021.10f" of |value|)
//
// digitComplement maps '0'..'9' → '9'..'0' (other bytes unchanged).
// Properties:
//   * Positives all sort after negatives (prefix '1' > '0').
//   * Within positives, magnitude order is preserved (zero-padded).
//   * Within negatives, larger magnitude → smaller complement →
//     sorts EARLIER, which is correct (-200 < -100).
//
// Format change is intentional. Indexes built with the old "%20.10f"
// scheme over numeric columns must be REINDEXed before use; this is
// a one-line `dbReindex()` per affected index. The previous output
// is silently wrong, so there is no safe migration that doesn't
// require rebuilding.
func valueToKeyBytes(v hbrt.Value) []byte {
	switch {
	case v.IsString():
		return []byte(v.AsString())
	case v.IsNumeric():
		return encodeNumericKey(v.AsNumDouble())
	case v.IsDate(), v.IsTimestamp():
		y, m, d := julianToDate(v.AsJulian())
		return []byte(fmt.Sprintf("%04d%02d%02d", y, m, d))
	case v.IsLogical():
		if v.AsBool() {
			return []byte("T")
		}
		return []byte("F")
	default:
		return []byte("")
	}
}

func encodeNumericKey(d float64) []byte {
	if d < 0 {
		// Magnitude with sign-aware Sprintf would interleave the
		// '-' inside the zero padding (e.g. "-0000000100.000000"),
		// which breaks lexicographic comparison. Format |value|
		// then prepend the sign + complement.
		mag := fmt.Sprintf("%021.10f", -d) // 21-byte unsigned magnitude
		b := make([]byte, 0, 22)
		b = append(b, '0')
		for i := 0; i < len(mag); i++ {
			c := mag[i]
			if c >= '0' && c <= '9' {
				b = append(b, '9'-(c-'0'))
			} else {
				b = append(b, c)
			}
		}
		return b
	}
	b := make([]byte, 0, 22)
	b = append(b, '1')
	b = append(b, fmt.Sprintf("%021.10f", d)...)
	return b
}

// Helper: find matching close parenthesis
func findMatchingParen(s string, openPos int) int {
	depth := 1
	for i := openPos + 1; i < len(s); i++ {
		if s[i] == '(' {
			depth++
		} else if s[i] == ')' {
			depth--
			if depth == 0 {
				return i
			}
		}
	}
	return -1
}

// Helper: find operator not inside parentheses
func findOperator(s string, op byte) int {
	depth := 0
	for i := len(s) - 1; i > 0; i-- {
		if s[i] == ')' {
			depth++
		} else if s[i] == '(' {
			depth--
		} else if s[i] == op && depth == 0 {
			return i
		}
	}
	return -1
}

// Helper: split comma-separated args respecting parentheses
func splitArgs(s string) []string {
	var args []string
	depth := 0
	start := 0
	for i := 0; i < len(s); i++ {
		if s[i] == '(' {
			depth++
		} else if s[i] == ')' {
			depth--
		} else if s[i] == ',' && depth == 0 {
			args = append(args, strings.TrimSpace(s[start:i]))
			start = i + 1
		}
	}
	args = append(args, strings.TrimSpace(s[start:]))
	return args
}

func parseIntIdx(s string) int {
	s = strings.TrimSpace(s)
	n := 0
	for _, c := range s {
		if c >= '0' && c <= '9' {
			n = n*10 + int(c-'0')
		}
	}
	return n
}

// formatKeyValue converts a Value to index key bytes.
func formatKeyValue(val hbrt.Value, fi hbrdd.FieldInfo) []byte {
	switch fi.Type {
	case 'C':
		s := val.AsString()
		// Pad to field length
		for len(s) < fi.Len {
			s += " "
		}
		return []byte(s[:fi.Len])
	case 'N':
		s := fmt.Sprintf("%*.*f", fi.Len, fi.Dec, val.AsNumDouble())
		return []byte(s)
	case 'D':
		return []byte(val.AsString())
	default:
		return []byte(val.AsString())
	}
}