fivedev/five
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: CDX mmap + internal node format (BE key-first) — 50K works

Browse Source 
CDX internal node format fix:
- Was: [child LE][recNo LE][key] (NTX-style)
- Now: [key][recNo BE][child BE] (correct CDX format)
- Fixes GoTop/Seek/Scan for large CDX files (50K+ records)

CDX mmap:
- syscall.Mmap on OpenIndex for zero-copy reads
- idx.readAt() helper: mmap slice or file fallback
- All ReadAt calls in Tag navigation replaced
- Close: munmap

CDX 50K benchmark (all counts correct):
  SEEK NAME 50K: 362ms (f=50000)
  SCAN 50K: 276ms (c=50000)
  SCOPE 35K: 238ms (c=35000)
  SEEK ID 50K: 320ms (f=50000)

CDX is slower than NTX due to bit-packed leaf decompression per page.
Cross-read test: 18/18 still PASS.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Charles KWON OhJun
2026-04-07 13:32:06 +09:00
parent a9600ad45c
commit 1b41384675
3 changed files with 153 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
hbrdd/cdx/cdx.go
View File

@@ -23,6 +23,7 @@ import (
"fmt"
"os"
"strings"
"syscall"
)
// CDX constants — matching Harbour.
@@ -250,9 +251,11 @@ type IntKeyEntry struct {
}
// DecodeIntKeys extracts keys from a CDX internal node.
// CDX internal format: [key: keyLen bytes][recNo: 4 BE][childPage: 4 BE]
// This is different from NTX which uses [child LE][recNo LE][key].
func DecodeIntKeys(data []byte, nKeys int, keyLen int) []IntKeyEntry {
entries := make([]IntKeyEntry, nKeys+1) // nKeys separators + 1 extra child
entrySize := 8 + keyLen
entries := make([]IntKeyEntry, nKeys+1) // nKeys separators + 1 trailing child
entrySize := keyLen + 8 // key + recNo(4BE) + child(4BE)
off := IntHeadSize
for i := 0; i <= nKeys; i++ {
@@ -260,12 +263,21 @@ func DecodeIntKeys(data []byte, nKeys int, keyLen int) []IntKeyEntry {
break
}
e := IntKeyEntry{
ChildPage: binary.LittleEndian.Uint32(data[off : off+4]),
RecNo: binary.LittleEndian.Uint32(data[off+4 : off+8]),
Key: make([]byte, keyLen),
Key: make([]byte, keyLen),
}
if i < nKeys {
copy(e.Key, data[off+8:off+8+keyLen])
copy(e.Key, data[off:off+keyLen])
e.RecNo = binary.BigEndian.Uint32(data[off+keyLen : off+keyLen+4])
e.ChildPage = binary.BigEndian.Uint32(data[off+keyLen+4 : off+keyLen+8])
} else {
// Trailing child — no key/recNo, just child pointer
// In CDX, the rightmost child is the child of the last separator entry
// Actually, for the last entry we only need the child pointer
// The previous entry's child was the LEFT child; we need RIGHT child
// stored at the position after last key entry
copy(e.Key, data[off:off+keyLen])
e.RecNo = binary.BigEndian.Uint32(data[off+keyLen : off+keyLen+4])
e.ChildPage = binary.BigEndian.Uint32(data[off+keyLen+4 : off+keyLen+8])
}
entries[i] = e
off += entrySize
@@ -278,8 +290,19 @@ func DecodeIntKeys(data []byte, nKeys int, keyLen int) []IntKeyEntry {
// Index represents an open CDX index file.
type Index struct {
file *os.File
tags []*Tag
file *os.File
tags []*Tag
mmapData []byte // mmap'd file for zero-copy reads
}
// readAt reads len(buf) bytes at offset — from mmap or file fallback.
func (idx *Index) readAt(buf []byte, offset int64) error {
if idx.mmapData != nil && offset >= 0 && int(offset)+len(buf) <= len(idx.mmapData) {
copy(buf, idx.mmapData[offset:offset+int64(len(buf))])
return nil
}
_, err := idx.file.ReadAt(buf, offset)
return err
}
// Tag represents one index tag within a CDX file.
@@ -317,6 +340,14 @@ func OpenIndex(path string) (*Index, error) {
idx := &Index{file: f}
// mmap for zero-copy reads
if fi, err2 := f.Stat(); err2 == nil && fi.Size() > 0 {
if data, err2 := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()),
syscall.PROT_READ, syscall.MAP_SHARED); err2 == nil {
idx.mmapData = data
}
}
// Read compound header (structural root at offset 0)
rootHdr, err := ReadTagHeader(f, 0)
if err != nil {
@@ -327,7 +358,7 @@ func OpenIndex(path string) (*Index, error) {
// Parse compound tag directory from the structural root's B-tree
// The structural index keys are 10-byte tag names, and each leaf entry
// points to the tag header at a specific file offset.
tagEntries := readCompoundTagList(f, rootHdr)
tagEntries := readCompoundTagList(idx, rootHdr)
for _, entry := range tagEntries {
tagHdr, err := ReadTagHeader(f, entry.offset)
@@ -363,6 +394,10 @@ func OpenIndex(path string) (*Index, error) {
// Close closes the CDX file.
func (idx *Index) Close() error {
if idx.mmapData != nil {
syscall.Munmap(idx.mmapData)
idx.mmapData = nil
}
return idx.file.Close()
}
@@ -406,7 +441,7 @@ type tagDirEntry struct {
// readCompoundTagList reads tag names and offsets from the structural root.
// CDX compound header: root page is a B-tree of tag entries.
// Each leaf key = 10-byte tag name, record number = page offset / 512.
func readCompoundTagList(f *os.File, rootHdr *TagHeader) []tagDirEntry {
func readCompoundTagList(idx *Index, rootHdr *TagHeader) []tagDirEntry {
var entries []tagDirEntry
if rootHdr.RootPtr == 0 {
return entries
@@ -414,7 +449,7 @@ func readCompoundTagList(f *os.File, rootHdr *TagHeader) []tagDirEntry {
// Read the root page of the structural index
pageData := make([]byte, 512)
_, err := f.ReadAt(pageData, int64(rootHdr.RootPtr))
err := idx.readAt(pageData, int64(rootHdr.RootPtr))
if err != nil {
return entries
}
@@ -431,7 +466,7 @@ func readCompoundTagList(f *os.File, rootHdr *TagHeader) []tagDirEntry {
// If compound leaf decoding didn't find entries, scan for tag headers
if len(entries) == 0 {
entries = scanCompoundLeaves(f, rootHdr)
entries = scanCompoundLeaves(idx, rootHdr)
}
return entries
@@ -440,10 +475,10 @@ func readCompoundTagList(f *os.File, rootHdr *TagHeader) []tagDirEntry {
// scanCompoundLeaves scans the CDX file for tag headers.
// CDX tag headers are at 0x400 (1024) byte boundaries.
// Each tag header is followed by a page with the key expression string.
func scanCompoundLeaves(f *os.File, rootHdr *TagHeader) []tagDirEntry {
func scanCompoundLeaves(idx *Index, rootHdr *TagHeader) []tagDirEntry {
var entries []tagDirEntry
fileInfo, err := f.Stat()
fileInfo, err := idx.file.Stat()
if err != nil {
return entries
}
@@ -458,8 +493,8 @@ func scanCompoundLeaves(f *os.File, rootHdr *TagHeader) []tagDirEntry {
// Tag headers are at 0x400 boundaries but NOT the compound root itself
for off := int64(0x400); off < fileSize; off += 0x200 {
buf := make([]byte, 0x400)
n, err := f.ReadAt(buf, off)
if err != nil || n < 0x200 {
err := idx.readAt(buf, off)
if err != nil {
continue
}
@@ -482,7 +517,7 @@ func scanCompoundLeaves(f *os.File, rootHdr *TagHeader) []tagDirEntry {
if keyExpr == "" {
// Key expression might be in the next page (+0x200 from header)
exprBuf := make([]byte, 256)
f.ReadAt(exprBuf, off+0x200)
idx.readAt(exprBuf, off+0x200)
for i := 0; i < len(exprBuf) && exprBuf[i] != 0; i++ {
keyExpr += string(exprBuf[i])
}
@@ -539,7 +574,7 @@ func (t *Tag) Seek(searchKey []byte) (uint32, bool) {
func (t *Tag) seekPage(pageOffset int64, searchKey []byte) (uint32, bool) {
buf := make([]byte, PageLen)
if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil {
if err := t.index.readAt(buf, pageOffset); err != nil {
t.tagEOF = true
return 0, false
}
@@ -618,7 +653,7 @@ func (t *Tag) GoTop() bool {
func (t *Tag) goLeftmost(pageOffset int64) bool {
buf := make([]byte, PageLen)
if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil {
if err := t.index.readAt(buf, pageOffset); err != nil {
return false
}
@@ -663,7 +698,7 @@ func (t *Tag) GoBottom() bool {
func (t *Tag) goRightmost(pageOffset int64) bool {
buf := make([]byte, PageLen)
if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil {
if err := t.index.readAt(buf, pageOffset); err != nil {
return false
}
@@ -710,7 +745,7 @@ func (t *Tag) SkipNext() bool {
keyIdx := t.stack[level].KeyIndex
buf := make([]byte, PageLen)
if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil {
if err := t.index.readAt(buf, pageOffset); err != nil {
t.tagEOF = true
return false
}
@@ -730,7 +765,7 @@ func (t *Tag) SkipNext() bool {
if hdr.RightPtr != 0 && hdr.RightPtr != 0xFFFFFFFF {
nextOff := int64(hdr.RightPtr)
buf2 := make([]byte, PageLen)
if _, err := t.index.file.ReadAt(buf2, nextOff); err != nil {
if err := t.index.readAt(buf2, nextOff); err != nil {
t.tagEOF = true
return false
}
@@ -761,7 +796,7 @@ func (t *Tag) SkipPrev() bool {
keyIdx := t.stack[level].KeyIndex
buf := make([]byte, PageLen)
if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil {
if err := t.index.readAt(buf, pageOffset); err != nil {
t.tagBOF = true
return false
}
@@ -781,7 +816,7 @@ func (t *Tag) SkipPrev() bool {
if hdr.LeftPtr != 0 && hdr.LeftPtr != 0xFFFFFFFF {
prevOff := int64(hdr.LeftPtr)
buf2 := make([]byte, PageLen)
if _, err := t.index.file.ReadAt(buf2, prevOff); err != nil {
if err := t.index.readAt(buf2, prevOff); err != nil {
t.tagBOF = true
return false
}