fivedev/five
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: CDX support + ORDSCOPE + cross-read Harbour compatibility

Browse Source 
CDX Integration:
- IndexEngine interface: common for NTX Index and CDX Tag
- OrderListAdd: auto-detects .cdx/.ntx extension, opens CDX tags
- decodeCompoundLeaf: proper bit-packed tag directory decoding
  (was stub falling through to scanCompoundLeaves with wrong names)
- CDX Tag: added KeyLen(), KeyExpr(), ForExpr(), IsDescending(), Close()
- CDX compound recNo = direct byte offset (not page number)

ORDSCOPE:
- SetScope/ClearScope/SetScopeTop/SetScopeBottom on DBFArea
- GoTopIndexed: seeks to scopeTop, validates within scopeBottom
- GoBottomIndexed: seeks to scopeBottom boundary
- SkipIndexed: stops at scope boundaries (top and bottom)
- OrdScope RTL function registered (nScope: 0=TOP, 1=BOTTOM)
- scopeKeyFromValue: converts Value to padded key bytes

Index Order Management:
- OrderListFocus: handles numeric order ("2" → order 2)
- SET ORDER TO n: gengo emits hbrt.NtoS for int-to-string conversion
- IndexOrd/OrdCount/OrdName/OrdKey: real implementations (were stubs)
- OrderCount/CurrentOrder/OrderName/OrderKeyExpr accessors on DBFArea
- ClearScope on order switch (prevents stale scope)

Cross-read test: Harbour-created CDX → Five reads, 20/20 items match:
  NAME/CITY/ID seek, ORDSCOPE count, GoTop/GoBottom all identical

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Charles KWON OhJun
2026-04-06 12:21:26 +09:00
parent 441d6c184f
commit 7e2a159b88
7 changed files with 613 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
hbrdd/cdx/cdx.go
View File

@@ -379,6 +379,9 @@ func (idx *Index) GetTag(i int) *Tag {
return nil
}
// Tags returns all tags in the CDX.
func (idx *Index) Tags() []*Tag { return idx.tags }
// FindTag returns a tag by name.
func (idx *Index) FindTag(name string) *Tag {
upper := strings.ToUpper(name)
@@ -498,12 +501,27 @@ func scanCompoundLeaves(f *os.File, rootHdr *TagHeader) []tagDirEntry {
}
// decodeCompoundLeaf decodes tag entries from a compound leaf page.
// Compound index uses the same bit-packed format as data leaves,
// with keyLen=10 (tag name) and recNo = page offset / PageLen.
func decodeCompoundLeaf(data []byte, nKeys int) []tagDirEntry {
if nKeys <= 0 || len(data) < ExtHeadSize {
return nil
}
// Use the standard leaf key decoder with keyLen=10 (compound tag name size)
hdr := DecodeLeafHeader(data)
keys := DecodeLeafKeys(data, hdr, 10)
var entries []tagDirEntry
// Compound index leaf format is simpler than data index
// Each entry: offset varies by CDX implementation
// For now return empty — scanCompoundLeaves handles it
_ = nKeys
for _, dk := range keys {
name := trimNull(dk.Key)
name = strings.TrimSpace(name)
if name == "" {
continue
}
// RecNo in compound index = direct byte offset to tag header
entries = append(entries, tagDirEntry{name: name, offset: int64(dk.RecNo)})
}
return entries
}
@@ -795,6 +813,21 @@ func (t *Tag) IsEOF() bool { return t.tagEOF }
// IsBOF returns true if before start.
func (t *Tag) IsBOF() bool { return t.tagBOF }
// KeyLen returns the key length.
func (t *Tag) KeyLen() int { return t.keyLen }
// KeyExpr returns the key expression string stored in the CDX header.
func (t *Tag) KeyExpr() string { return t.header.KeyExpr }
// ForExpr returns the FOR condition expression.
func (t *Tag) ForExpr() string { return t.header.ForExpr }
// IsDescending returns true if the tag sorts in descending order.
func (t *Tag) IsDescending() bool { return t.header.Descending }
// Close is a no-op for tags (the parent Index owns the file).
func (t *Tag) Close() error { return nil }
// --- Helpers ---
func trimNull(b []byte) string {

332
hbrdd/dbf/indexer.go
View File

@@ -10,6 +10,7 @@ import (
"bytes"
"five/hbrt"
"five/hbrdd"
"five/hbrdd/cdx"
"five/hbrdd/ntx"
"fmt"
"os"
@@ -18,13 +19,31 @@ import (
"strings"
)
// IndexEngine is the common interface for NTX Index and CDX Tag.
type IndexEngine interface {
Seek(searchKey []byte) (uint32, bool)
GoTop() bool
GoBottom() bool
SkipNext() bool
SkipPrev() bool
CurRecNo() uint32
CurKey() []byte
IsEOF() bool
IsBOF() bool
KeyLen() int
Close() error
}
// indexState holds active index state for a DBFArea.
type indexState struct {
indexes []*ntx.Index // open NTX index files
indexes []IndexEngine // open NTX/CDX index engines
names []string // index file paths
tags []string // tag names (for display)
current int // active index (-1 = natural order)
keyExprs []string // key expressions for each index
// Scope support
scopeTop []byte // top scope key (nil = no scope)
scopeBottom []byte // bottom scope key (nil = no scope)
}
// ensureIndexState initializes the index state if nil.
@@ -117,14 +136,40 @@ func (a *DBFArea) OrderCreate(params hbrdd.OrderCreateParams) error {
return nil
}
// OrderListAdd opens an existing index file.
// OrderListAdd opens an existing index file (NTX single-order or CDX compound).
func (a *DBFArea) OrderListAdd(path string) error {
a.ensureIndexState()
// Auto-detect extension: try .cdx first, then .ntx
if !strings.Contains(filepath.Base(path), ".") {
path += ".ntx"
if _, err := os.Stat(path + ".cdx"); err == nil {
path += ".cdx"
} else {
path += ".ntx"
}
}
ext := strings.ToLower(filepath.Ext(path))
if ext == ".cdx" {
// CDX compound index — opens all tags
ci, err := cdx.OpenIndex(path)
if err != nil {
return fmt.Errorf("open CDX failed: %w", err)
}
for _, tag := range ci.Tags() {
a.idxState.indexes = append(a.idxState.indexes, tag)
a.idxState.names = append(a.idxState.names, path)
a.idxState.tags = append(a.idxState.tags, tag.Name)
a.idxState.keyExprs = append(a.idxState.keyExprs, tag.KeyExpr())
}
if len(ci.Tags()) > 0 {
a.idxState.current = len(a.idxState.indexes) - len(ci.Tags()) // first tag
}
return nil
}
// NTX single index
idx, err := ntx.OpenIndex(path)
if err != nil {
return fmt.Errorf("open index failed: %w", err)
@@ -151,31 +196,68 @@ func (a *DBFArea) OrderListClear() error {
return nil
}
// OrderListFocus sets the active index by tag name or number.
// OrderListFocus sets the active index by tag name, number, or file name.
// Harbour: OrdSetFocus(nOrder) or OrdSetFocus("tagName")
func (a *DBFArea) OrderListFocus(tagName string) error {
a.ensureIndexState()
if tagName == "" || tagName == "0" {
a.idxState.current = -1 // natural order
a.ClearScope()
return nil
}
upper := strings.ToUpper(tagName)
for i, name := range a.idxState.tags {
if strings.ToUpper(name) == upper {
a.idxState.current = i
// Try as numeric order (1-based)
if n, err := parseOrderNum(tagName); err == nil {
if n == 0 {
a.idxState.current = -1
a.ClearScope()
return nil
}
if n >= 1 && n <= len(a.idxState.indexes) {
a.idxState.current = n - 1
a.ClearScope()
return nil
}
}
// Try by file name
upper := strings.ToUpper(tagName)
// Match by tag name
for i, name := range a.idxState.tags {
if strings.ToUpper(name) == upper {
a.idxState.current = i
a.ClearScope()
return nil
}
}
// Match by file name
for i, name := range a.idxState.names {
base := strings.ToUpper(filepath.Base(name))
if base == upper || strings.TrimSuffix(base, ".NTX") == upper {
ext := strings.ToUpper(filepath.Ext(name))
if base == upper || strings.TrimSuffix(base, ext) == upper {
a.idxState.current = i
a.ClearScope()
return nil
}
}
return fmt.Errorf("index not found: %s", tagName)
}
// parseOrderNum tries to parse a string as a positive integer (order number).
func parseOrderNum(s string) (int, error) {
s = strings.TrimSpace(s)
if len(s) == 0 {
return 0, fmt.Errorf("empty")
}
n := 0
for _, c := range s {
if c < '0' || c > '9' {
return 0, fmt.Errorf("not a number")
}
n = n*10 + int(c-'0')
}
return n, nil
}
// OrderListRebuild rebuilds all indexes.
// Harbour: ORDLISTREBUILD / REINDEX — recreates all open indexes from current data.
func (a *DBFArea) OrderListRebuild() error {
@@ -344,25 +426,83 @@ func (a *DBFArea) Seek(key hbrt.Value, softSeek bool, findLast bool) (bool, erro
}
// GoTopIndexed positions at the first key in the active index.
// Harbour: if SCOPE is set, positions at the first key >= scopeTop.
func (a *DBFArea) GoTopIndexed() error {
if a.idxState == nil || a.idxState.current < 0 {
return a.GoTop()
}
idx := a.idxState.indexes[a.idxState.current]
if a.idxState.scopeTop != nil {
// Seek to scope top boundary
recNo, _ := idx.Seek(a.idxState.scopeTop)
if recNo == 0 || idx.IsEOF() {
rc, _ := a.RecCount()
a.FEof = true
return a.GoTo(rc + 1)
}
// Check if within bottom scope
if a.idxState.scopeBottom != nil {
if bytes.Compare(idx.CurKey(), a.idxState.scopeBottom) > 0 {
rc, _ := a.RecCount()
a.FEof = true
return a.GoTo(rc + 1)
}
}
return a.GoTo(idx.CurRecNo())
}
idx.GoTop()
if idx.IsEOF() {
rc, _ := a.RecCount()
a.FEof = true
return a.GoTo(rc + 1)
}
return a.GoTo(idx.CurRecNo())
}
// GoBottomIndexed positions at the last key in the active index.
// Harbour: if SCOPE is set, positions at the last key <= scopeBottom.
func (a *DBFArea) GoBottomIndexed() error {
if a.idxState == nil || a.idxState.current < 0 {
return a.GoBottom()
}
idx := a.idxState.indexes[a.idxState.current]
if a.idxState.scopeBottom != nil {
// Seek to scope bottom boundary
_, exact := idx.Seek(a.idxState.scopeBottom)
if idx.IsEOF() {
// All keys less than bottom scope — go to physical bottom
idx.GoBottom()
} else if !exact {
// Positioned past bottom — go back one
idx.SkipPrev()
} else {
// Exact match — skip forward to last matching key, then position there
for {
idx.SkipNext()
if idx.IsEOF() || bytes.Compare(idx.CurKey(), a.idxState.scopeBottom) > 0 {
idx.SkipPrev()
break
}
}
}
if idx.IsBOF() || idx.IsEOF() {
a.FBof = true
return a.GoTo(1)
}
// Verify within top scope
if a.idxState.scopeTop != nil {
if bytes.Compare(idx.CurKey(), a.idxState.scopeTop) < 0 {
a.FEof = true
rc, _ := a.RecCount()
return a.GoTo(rc + 1)
}
}
return a.GoTo(idx.CurRecNo())
}
idx.GoBottom()
if idx.IsBOF() {
return a.GoTo(1)
@@ -371,11 +511,13 @@ func (a *DBFArea) GoBottomIndexed() error {
}
// SkipIndexed skips using the active index order.
// Harbour: respects SCOPE boundaries — stops at scope edges.
func (a *DBFArea) SkipIndexed(count int64) error {
if a.idxState == nil || a.idxState.current < 0 {
return a.Skip(count)
}
idx := a.idxState.indexes[a.idxState.current]
hasScope := a.idxState.scopeTop != nil || a.idxState.scopeBottom != nil
if count > 0 {
for i := int64(0); i < count; i++ {
@@ -386,18 +528,188 @@ func (a *DBFArea) SkipIndexed(count int64) error {
a.FEof = true
return nil
}
// Check bottom scope
if hasScope && a.idxState.scopeBottom != nil {
if bytes.Compare(idx.CurKey(), a.idxState.scopeBottom) > 0 {
rc, _ := a.RecCount()
a.GoTo(rc + 1)
a.FEof = true
return nil
}
}
}
} else if count < 0 {
for i := int64(0); i > count; i-- {
idx.SkipPrev()
if idx.IsBOF() {
a.FBof = true
// Stay at first record in scope
if a.idxState.scopeTop != nil {
idx.Seek(a.idxState.scopeTop)
} else {
idx.GoTop()
}
if !idx.IsEOF() {
return a.GoTo(idx.CurRecNo())
}
return a.GoTo(1)
}
// Check top scope
if hasScope && a.idxState.scopeTop != nil {
if bytes.Compare(idx.CurKey(), a.idxState.scopeTop) < 0 {
a.FBof = true
idx.Seek(a.idxState.scopeTop)
if !idx.IsEOF() {
return a.GoTo(idx.CurRecNo())
}
return a.GoTo(1)
}
}
}
}
return a.GoTo(idx.CurRecNo())
}
// --- Scope support (ORDSCOPE) ---
// SetScope sets top and/or bottom scope boundaries for the active index.
// Harbour: OrdScope(TOPSCOPE, val) / OrdScope(BOTTOMSCOPE, val)
// Pass zero-value hbrt.Value{} (not MakeNil) to skip setting that boundary.
func (a *DBFArea) SetScope(top, bottom hbrt.Value) error {
a.ensureIndexState()
if a.idxState.current < 0 {
return fmt.Errorf("no active index")
}
idx := a.idxState.indexes[a.idxState.current]
keyLen := idx.KeyLen()
if !top.IsNil() && top.Type() != 0 {
a.idxState.scopeTop = scopeKeyFromValue(top, keyLen)
}
if !bottom.IsNil() && bottom.Type() != 0 {
a.idxState.scopeBottom = scopeKeyFromValue(bottom, keyLen)
}
return nil
}
// SetScopeTop sets only the top scope.
func (a *DBFArea) SetScopeTop(val hbrt.Value) {
a.ensureIndexState()
if a.idxState.current < 0 {
return
}
keyLen := a.idxState.indexes[a.idxState.current].KeyLen()
a.idxState.scopeTop = scopeKeyFromValue(val, keyLen)
}
// SetScopeBottom sets only the bottom scope.
func (a *DBFArea) SetScopeBottom(val hbrt.Value) {
a.ensureIndexState()
if a.idxState.current < 0 {
return
}
keyLen := a.idxState.indexes[a.idxState.current].KeyLen()
a.idxState.scopeBottom = scopeKeyFromValue(val, keyLen)
}
// ClearScope removes all scope boundaries.
func (a *DBFArea) ClearScope() error {
if a.idxState != nil {
a.idxState.scopeTop = nil
a.idxState.scopeBottom = nil
}
return nil
}
// ClearScopeTop removes only the top scope boundary.
func (a *DBFArea) ClearScopeTop() {
if a.idxState != nil {
a.idxState.scopeTop = nil
}
}
// ClearScopeBottom removes only the bottom scope boundary.
func (a *DBFArea) ClearScopeBottom() {
if a.idxState != nil {
a.idxState.scopeBottom = nil
}
}
// GetScopeTop returns the current top scope key (nil if none).
func (a *DBFArea) GetScopeTop() []byte {
if a.idxState != nil {
return a.idxState.scopeTop
}
return nil
}
// GetScopeBottom returns the current bottom scope key (nil if none).
func (a *DBFArea) GetScopeBottom() []byte {
if a.idxState != nil {
return a.idxState.scopeBottom
}
return nil
}
// scopeKeyFromValue converts a Harbour Value to a scope key byte slice.
func scopeKeyFromValue(v hbrt.Value, keyLen int) []byte {
var key []byte
if v.IsString() {
key = []byte(v.AsString())
} else if v.IsNumeric() {
key = []byte(fmt.Sprintf("%*d", keyLen, v.AsNumInt()))
} else {
key = []byte(v.AsString())
}
// Pad to keyLen
if len(key) < keyLen {
padded := make([]byte, keyLen)
copy(padded, key)
for i := len(key); i < keyLen; i++ {
padded[i] = ' '
}
return padded
}
if len(key) > keyLen {
return key[:keyLen]
}
return key
}
// --- Index info accessors ---
// IndexCount returns the number of open indexes.
func (a *DBFArea) IndexCount() int {
if a.idxState == nil {
return 0
}
return len(a.idxState.indexes)
}
// CurrentOrder returns the 1-based current order number (0 = natural).
func (a *DBFArea) CurrentOrder() int {
if a.idxState == nil || a.idxState.current < 0 {
return 0
}
return a.idxState.current + 1
}
// OrderName returns the tag name for order n (1-based).
func (a *DBFArea) OrderName(n int) string {
if a.idxState == nil || n < 1 || n > len(a.idxState.tags) {
return ""
}
return a.idxState.tags[n-1]
}
// OrderKeyExpr returns the key expression for order n (1-based).
func (a *DBFArea) OrderKeyExpr(n int) string {
if a.idxState == nil || n < 1 || n > len(a.idxState.keyExprs) {
return ""
}
return a.idxState.keyExprs[n-1]
}
// evalKeyExpr evaluates an index key expression for a given record.
// Supports: field names, UPPER(), LOWER(), LTRIM(), RTRIM(), ALLTRIM(),
// STR(), DTOS(), SUBSTR(), LEFT(), RIGHT(), PADL(), PADR(),