fix: CDX mmap + internal node format (BE key-first) — 50K works

CDX internal node format fix: - Was: [child LE][recNo LE][key] (NTX-style) - Now: [key][recNo BE][child BE] (correct CDX format) - Fixes GoTop/Seek/Scan for large CDX files (50K+ records) CDX mmap: - syscall.Mmap on OpenIndex for zero-copy reads - idx.readAt() helper: mmap slice or file fallback - All ReadAt calls in Tag navigation replaced - Close: munmap CDX 50K benchmark (all counts correct): SEEK NAME 50K: 362ms (f=50000) SCAN 50K: 276ms (c=50000) SCOPE 35K: 238ms (c=35000) SEEK ID 50K: 320ms (f=50000) CDX is slower than NTX due to bit-packed leaf decompression per page. Cross-read test: 18/18 still PASS. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-07 13:32:06 +09:00
parent a9600ad45c
commit 1b41384675
3 changed files with 153 additions and 24 deletions
--- a/examples/bench_cdx_read.prg
+++ b/examples/bench_cdx_read.prg
@@ -0,0 +1,66 @@
+// CDX Read Benchmark — opens Harbour-created CDX, tests seek/scan/scope
+PROCEDURE Main()
+   LOCAL i, nStart, nEnd, nCount
+
+   // Open Harbour-created files
+   USE "cdx_bench" NEW
+   SET INDEX TO "cdx_bench.cdx"
+
+   ? "TAGS=" + LTrim(Str(OrdCount()))
+
+   // SEEK 50K by NAME
+   SET ORDER TO 3
+   nStart := Seconds()
+   nCount := 0
+   FOR i := 1 TO 50000
+      SEEK PadR("Name_" + PadL(LTrim(Str(i)), 5, "0"), 20)
+      IF Found()
+         nCount++
+      ENDIF
+   NEXT
+   nEnd := Seconds()
+   ? "SEEK_NAME=" + LTrim(Str(Int((nEnd-nStart)*1000))) + "ms f=" + LTrim(Str(nCount))
+
+   // SCAN 50K
+   SET ORDER TO 3
+   GO TOP
+   nStart := Seconds()
+   nCount := 0
+   DO WHILE !EOF()
+      nCount++
+      SKIP
+   ENDDO
+   nEnd := Seconds()
+   ? "SCAN=" + LTrim(Str(Int((nEnd-nStart)*1000))) + "ms c=" + LTrim(Str(nCount))
+
+   // ORDSCOPE on CITY
+   SET ORDER TO 1
+   OrdScope(0, PadR("London", 15))
+   OrdScope(1, PadR("Seoul", 15))
+   nStart := Seconds()
+   GO TOP
+   nCount := 0
+   DO WHILE !EOF()
+      nCount++
+      SKIP
+   ENDDO
+   nEnd := Seconds()
+   ? "SCOPE=" + LTrim(Str(Int((nEnd-nStart)*1000))) + "ms c=" + LTrim(Str(nCount))
+   OrdScope(0, NIL)
+   OrdScope(1, NIL)
+
+   // SEEK by ID
+   SET ORDER TO 2
+   nStart := Seconds()
+   nCount := 0
+   FOR i := 1 TO 50000
+      SEEK Str(i, 8)
+      IF Found()
+         nCount++
+      ENDIF
+   NEXT
+   nEnd := Seconds()
+   ? "SEEK_ID=" + LTrim(Str(Int((nEnd-nStart)*1000))) + "ms f=" + LTrim(Str(nCount))
+
+   CLOSE ALL
+RETURN
--- a/examples/debug_cdx50k.prg
+++ b/examples/debug_cdx50k.prg
@@ -0,0 +1,28 @@
+PROCEDURE Main()
+   USE "cdx_bench" NEW
+   SET INDEX TO "cdx_bench.cdx"
+
+   ? "Tags:", OrdCount()
+   ? "Order1:", OrdName(1)
+   ? "Order2:", OrdName(2)
+   ? "Order3:", OrdName(3)
+
+   SET ORDER TO 3
+   ? "Current:", IndexOrd()
+   GO TOP
+   ? "Top:", RecNo(), EOF(), BOF()
+   ? "Name:", RTrim(FieldGet(2))
+   SKIP
+   ? "Skip:", RecNo(), EOF()
+
+   SET ORDER TO 1
+   GO TOP
+   ? "City top:", RecNo(), RTrim(FieldGet(3)), EOF()
+
+   SET ORDER TO 0
+   GO TOP
+   ? "Natural:", RecNo(), LTrim(Str(FieldGet(1)))
+   ? "RC:", RecCount()
+
+   CLOSE ALL
+RETURN
--- a/hbrdd/cdx/cdx.go
+++ b/hbrdd/cdx/cdx.go
@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"syscall"
 )

 // CDX constants — matching Harbour.
@@ -250,9 +251,11 @@ type IntKeyEntry struct {
 }

 // DecodeIntKeys extracts keys from a CDX internal node.
+// CDX internal format: [key: keyLen bytes][recNo: 4 BE][childPage: 4 BE]
+// This is different from NTX which uses [child LE][recNo LE][key].
 func DecodeIntKeys(data []byte, nKeys int, keyLen int) []IntKeyEntry {
-	entries := make([]IntKeyEntry, nKeys+1) // nKeys separators + 1 extra child
-	entrySize := 8 + keyLen
+	entries := make([]IntKeyEntry, nKeys+1) // nKeys separators + 1 trailing child
+	entrySize := keyLen + 8                 // key + recNo(4BE) + child(4BE)
 	off := IntHeadSize

 	for i := 0; i <= nKeys; i++ {
@@ -260,12 +263,21 @@ func DecodeIntKeys(data []byte, nKeys int, keyLen int) []IntKeyEntry {
 			break
 		}
 		e := IntKeyEntry{
-			ChildPage: binary.LittleEndian.Uint32(data[off : off+4]),
-			RecNo:     binary.LittleEndian.Uint32(data[off+4 : off+8]),
-			Key:       make([]byte, keyLen),
+			Key: make([]byte, keyLen),
 		}
 		if i < nKeys {
-			copy(e.Key, data[off+8:off+8+keyLen])
+			copy(e.Key, data[off:off+keyLen])
+			e.RecNo = binary.BigEndian.Uint32(data[off+keyLen : off+keyLen+4])
+			e.ChildPage = binary.BigEndian.Uint32(data[off+keyLen+4 : off+keyLen+8])
+		} else {
+			// Trailing child — no key/recNo, just child pointer
+			// In CDX, the rightmost child is the child of the last separator entry
+			// Actually, for the last entry we only need the child pointer
+			// The previous entry's child was the LEFT child; we need RIGHT child
+			// stored at the position after last key entry
+			copy(e.Key, data[off:off+keyLen])
+			e.RecNo = binary.BigEndian.Uint32(data[off+keyLen : off+keyLen+4])
+			e.ChildPage = binary.BigEndian.Uint32(data[off+keyLen+4 : off+keyLen+8])
 		}
 		entries[i] = e
 		off += entrySize
@@ -278,8 +290,19 @@ func DecodeIntKeys(data []byte, nKeys int, keyLen int) []IntKeyEntry {

 // Index represents an open CDX index file.
 type Index struct {
-	file *os.File
-	tags []*Tag
+	file     *os.File
+	tags     []*Tag
+	mmapData []byte // mmap'd file for zero-copy reads
+}
+
+// readAt reads len(buf) bytes at offset — from mmap or file fallback.
+func (idx *Index) readAt(buf []byte, offset int64) error {
+	if idx.mmapData != nil && offset >= 0 && int(offset)+len(buf) <= len(idx.mmapData) {
+		copy(buf, idx.mmapData[offset:offset+int64(len(buf))])
+		return nil
+	}
+	_, err := idx.file.ReadAt(buf, offset)
+	return err
 }

 // Tag represents one index tag within a CDX file.
@@ -317,6 +340,14 @@ func OpenIndex(path string) (*Index, error) {

 	idx := &Index{file: f}

+	// mmap for zero-copy reads
+	if fi, err2 := f.Stat(); err2 == nil && fi.Size() > 0 {
+		if data, err2 := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()),
+			syscall.PROT_READ, syscall.MAP_SHARED); err2 == nil {
+			idx.mmapData = data
+		}
+	}
+
 	// Read compound header (structural root at offset 0)
 	rootHdr, err := ReadTagHeader(f, 0)
 	if err != nil {
@@ -327,7 +358,7 @@ func OpenIndex(path string) (*Index, error) {
 	// Parse compound tag directory from the structural root's B-tree
 	// The structural index keys are 10-byte tag names, and each leaf entry
 	// points to the tag header at a specific file offset.
-	tagEntries := readCompoundTagList(f, rootHdr)
+	tagEntries := readCompoundTagList(idx, rootHdr)

 	for _, entry := range tagEntries {
 		tagHdr, err := ReadTagHeader(f, entry.offset)
@@ -363,6 +394,10 @@ func OpenIndex(path string) (*Index, error) {

 // Close closes the CDX file.
 func (idx *Index) Close() error {
+	if idx.mmapData != nil {
+		syscall.Munmap(idx.mmapData)
+		idx.mmapData = nil
+	}
 	return idx.file.Close()
 }

@@ -406,7 +441,7 @@ type tagDirEntry struct {
 // readCompoundTagList reads tag names and offsets from the structural root.
 // CDX compound header: root page is a B-tree of tag entries.
 // Each leaf key = 10-byte tag name, record number = page offset / 512.
-func readCompoundTagList(f *os.File, rootHdr *TagHeader) []tagDirEntry {
+func readCompoundTagList(idx *Index, rootHdr *TagHeader) []tagDirEntry {
 	var entries []tagDirEntry
 	if rootHdr.RootPtr == 0 {
 		return entries
@@ -414,7 +449,7 @@ func readCompoundTagList(f *os.File, rootHdr *TagHeader) []tagDirEntry {

 	// Read the root page of the structural index
 	pageData := make([]byte, 512)
-	_, err := f.ReadAt(pageData, int64(rootHdr.RootPtr))
+	err := idx.readAt(pageData, int64(rootHdr.RootPtr))
 	if err != nil {
 		return entries
 	}
@@ -431,7 +466,7 @@ func readCompoundTagList(f *os.File, rootHdr *TagHeader) []tagDirEntry {

 	// If compound leaf decoding didn't find entries, scan for tag headers
 	if len(entries) == 0 {
-		entries = scanCompoundLeaves(f, rootHdr)
+		entries = scanCompoundLeaves(idx, rootHdr)
 	}

 	return entries
@@ -440,10 +475,10 @@ func readCompoundTagList(f *os.File, rootHdr *TagHeader) []tagDirEntry {
 // scanCompoundLeaves scans the CDX file for tag headers.
 // CDX tag headers are at 0x400 (1024) byte boundaries.
 // Each tag header is followed by a page with the key expression string.
-func scanCompoundLeaves(f *os.File, rootHdr *TagHeader) []tagDirEntry {
+func scanCompoundLeaves(idx *Index, rootHdr *TagHeader) []tagDirEntry {
 	var entries []tagDirEntry

-	fileInfo, err := f.Stat()
+	fileInfo, err := idx.file.Stat()
 	if err != nil {
 		return entries
 	}
@@ -458,8 +493,8 @@ func scanCompoundLeaves(f *os.File, rootHdr *TagHeader) []tagDirEntry {
 	// Tag headers are at 0x400 boundaries but NOT the compound root itself
 	for off := int64(0x400); off < fileSize; off += 0x200 {
 		buf := make([]byte, 0x400)
-		n, err := f.ReadAt(buf, off)
-		if err != nil || n < 0x200 {
+		err := idx.readAt(buf, off)
+		if err != nil {
 			continue
 		}

@@ -482,7 +517,7 @@ func scanCompoundLeaves(f *os.File, rootHdr *TagHeader) []tagDirEntry {
 		if keyExpr == "" {
 			// Key expression might be in the next page (+0x200 from header)
 			exprBuf := make([]byte, 256)
-			f.ReadAt(exprBuf, off+0x200)
+			idx.readAt(exprBuf, off+0x200)
 			for i := 0; i < len(exprBuf) && exprBuf[i] != 0; i++ {
 				keyExpr += string(exprBuf[i])
 			}
@@ -539,7 +574,7 @@ func (t *Tag) Seek(searchKey []byte) (uint32, bool) {

 func (t *Tag) seekPage(pageOffset int64, searchKey []byte) (uint32, bool) {
 	buf := make([]byte, PageLen)
-	if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil {
+	if err := t.index.readAt(buf, pageOffset); err != nil {
 		t.tagEOF = true
 		return 0, false
 	}
@@ -618,7 +653,7 @@ func (t *Tag) GoTop() bool {

 func (t *Tag) goLeftmost(pageOffset int64) bool {
 	buf := make([]byte, PageLen)
-	if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil {
+	if err := t.index.readAt(buf, pageOffset); err != nil {
 		return false
 	}

@@ -663,7 +698,7 @@ func (t *Tag) GoBottom() bool {

 func (t *Tag) goRightmost(pageOffset int64) bool {
 	buf := make([]byte, PageLen)
-	if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil {
+	if err := t.index.readAt(buf, pageOffset); err != nil {
 		return false
 	}

@@ -710,7 +745,7 @@ func (t *Tag) SkipNext() bool {
 	keyIdx := t.stack[level].KeyIndex

 	buf := make([]byte, PageLen)
-	if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil {
+	if err := t.index.readAt(buf, pageOffset); err != nil {
 		t.tagEOF = true
 		return false
 	}
@@ -730,7 +765,7 @@ func (t *Tag) SkipNext() bool {
 	if hdr.RightPtr != 0 && hdr.RightPtr != 0xFFFFFFFF {
 		nextOff := int64(hdr.RightPtr)
 		buf2 := make([]byte, PageLen)
-		if _, err := t.index.file.ReadAt(buf2, nextOff); err != nil {
+		if err := t.index.readAt(buf2, nextOff); err != nil {
 			t.tagEOF = true
 			return false
 		}
@@ -761,7 +796,7 @@ func (t *Tag) SkipPrev() bool {
 	keyIdx := t.stack[level].KeyIndex

 	buf := make([]byte, PageLen)
-	if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil {
+	if err := t.index.readAt(buf, pageOffset); err != nil {
 		t.tagBOF = true
 		return false
 	}
@@ -781,7 +816,7 @@ func (t *Tag) SkipPrev() bool {
 	if hdr.LeftPtr != 0 && hdr.LeftPtr != 0xFFFFFFFF {
 		prevOff := int64(hdr.LeftPtr)
 		buf2 := make([]byte, PageLen)
-		if _, err := t.index.file.ReadAt(buf2, prevOff); err != nil {
+		if err := t.index.readAt(buf2, prevOff); err != nil {
 			t.tagBOF = true
 			return false
 		}