From 1b41384675f04967d045639ec74c9ac565e02d2f Mon Sep 17 00:00:00 2001 From: Charles KWON OhJun Date: Tue, 7 Apr 2026 13:32:06 +0900 Subject: [PATCH] =?UTF-8?q?fix:=20CDX=20mmap=20+=20internal=20node=20forma?= =?UTF-8?q?t=20(BE=20key-first)=20=E2=80=94=2050K=20works?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CDX internal node format fix: - Was: [child LE][recNo LE][key] (NTX-style) - Now: [key][recNo BE][child BE] (correct CDX format) - Fixes GoTop/Seek/Scan for large CDX files (50K+ records) CDX mmap: - syscall.Mmap on OpenIndex for zero-copy reads - idx.readAt() helper: mmap slice or file fallback - All ReadAt calls in Tag navigation replaced - Close: munmap CDX 50K benchmark (all counts correct): SEEK NAME 50K: 362ms (f=50000) SCAN 50K: 276ms (c=50000) SCOPE 35K: 238ms (c=35000) SEEK ID 50K: 320ms (f=50000) CDX is slower than NTX due to bit-packed leaf decompression per page. Cross-read test: 18/18 still PASS. Co-Authored-By: Claude Opus 4.6 (1M context) --- examples/bench_cdx_read.prg | 66 +++++++++++++++++++++++++++++ examples/debug_cdx50k.prg | 28 +++++++++++++ hbrdd/cdx/cdx.go | 83 ++++++++++++++++++++++++++----------- 3 files changed, 153 insertions(+), 24 deletions(-) create mode 100644 examples/bench_cdx_read.prg create mode 100644 examples/debug_cdx50k.prg diff --git a/examples/bench_cdx_read.prg b/examples/bench_cdx_read.prg new file mode 100644 index 0000000..a08c3ae --- /dev/null +++ b/examples/bench_cdx_read.prg @@ -0,0 +1,66 @@ +// CDX Read Benchmark — opens Harbour-created CDX, tests seek/scan/scope +PROCEDURE Main() + LOCAL i, nStart, nEnd, nCount + + // Open Harbour-created files + USE "cdx_bench" NEW + SET INDEX TO "cdx_bench.cdx" + + ? "TAGS=" + LTrim(Str(OrdCount())) + + // SEEK 50K by NAME + SET ORDER TO 3 + nStart := Seconds() + nCount := 0 + FOR i := 1 TO 50000 + SEEK PadR("Name_" + PadL(LTrim(Str(i)), 5, "0"), 20) + IF Found() + nCount++ + ENDIF + NEXT + nEnd := Seconds() + ? "SEEK_NAME=" + LTrim(Str(Int((nEnd-nStart)*1000))) + "ms f=" + LTrim(Str(nCount)) + + // SCAN 50K + SET ORDER TO 3 + GO TOP + nStart := Seconds() + nCount := 0 + DO WHILE !EOF() + nCount++ + SKIP + ENDDO + nEnd := Seconds() + ? "SCAN=" + LTrim(Str(Int((nEnd-nStart)*1000))) + "ms c=" + LTrim(Str(nCount)) + + // ORDSCOPE on CITY + SET ORDER TO 1 + OrdScope(0, PadR("London", 15)) + OrdScope(1, PadR("Seoul", 15)) + nStart := Seconds() + GO TOP + nCount := 0 + DO WHILE !EOF() + nCount++ + SKIP + ENDDO + nEnd := Seconds() + ? "SCOPE=" + LTrim(Str(Int((nEnd-nStart)*1000))) + "ms c=" + LTrim(Str(nCount)) + OrdScope(0, NIL) + OrdScope(1, NIL) + + // SEEK by ID + SET ORDER TO 2 + nStart := Seconds() + nCount := 0 + FOR i := 1 TO 50000 + SEEK Str(i, 8) + IF Found() + nCount++ + ENDIF + NEXT + nEnd := Seconds() + ? "SEEK_ID=" + LTrim(Str(Int((nEnd-nStart)*1000))) + "ms f=" + LTrim(Str(nCount)) + + CLOSE ALL +RETURN diff --git a/examples/debug_cdx50k.prg b/examples/debug_cdx50k.prg new file mode 100644 index 0000000..2cb5214 --- /dev/null +++ b/examples/debug_cdx50k.prg @@ -0,0 +1,28 @@ +PROCEDURE Main() + USE "cdx_bench" NEW + SET INDEX TO "cdx_bench.cdx" + + ? "Tags:", OrdCount() + ? "Order1:", OrdName(1) + ? "Order2:", OrdName(2) + ? "Order3:", OrdName(3) + + SET ORDER TO 3 + ? "Current:", IndexOrd() + GO TOP + ? "Top:", RecNo(), EOF(), BOF() + ? "Name:", RTrim(FieldGet(2)) + SKIP + ? "Skip:", RecNo(), EOF() + + SET ORDER TO 1 + GO TOP + ? "City top:", RecNo(), RTrim(FieldGet(3)), EOF() + + SET ORDER TO 0 + GO TOP + ? "Natural:", RecNo(), LTrim(Str(FieldGet(1))) + ? "RC:", RecCount() + + CLOSE ALL +RETURN diff --git a/hbrdd/cdx/cdx.go b/hbrdd/cdx/cdx.go index 37b5509..0ad4cf5 100644 --- a/hbrdd/cdx/cdx.go +++ b/hbrdd/cdx/cdx.go @@ -23,6 +23,7 @@ import ( "fmt" "os" "strings" + "syscall" ) // CDX constants — matching Harbour. @@ -250,9 +251,11 @@ type IntKeyEntry struct { } // DecodeIntKeys extracts keys from a CDX internal node. +// CDX internal format: [key: keyLen bytes][recNo: 4 BE][childPage: 4 BE] +// This is different from NTX which uses [child LE][recNo LE][key]. func DecodeIntKeys(data []byte, nKeys int, keyLen int) []IntKeyEntry { - entries := make([]IntKeyEntry, nKeys+1) // nKeys separators + 1 extra child - entrySize := 8 + keyLen + entries := make([]IntKeyEntry, nKeys+1) // nKeys separators + 1 trailing child + entrySize := keyLen + 8 // key + recNo(4BE) + child(4BE) off := IntHeadSize for i := 0; i <= nKeys; i++ { @@ -260,12 +263,21 @@ func DecodeIntKeys(data []byte, nKeys int, keyLen int) []IntKeyEntry { break } e := IntKeyEntry{ - ChildPage: binary.LittleEndian.Uint32(data[off : off+4]), - RecNo: binary.LittleEndian.Uint32(data[off+4 : off+8]), - Key: make([]byte, keyLen), + Key: make([]byte, keyLen), } if i < nKeys { - copy(e.Key, data[off+8:off+8+keyLen]) + copy(e.Key, data[off:off+keyLen]) + e.RecNo = binary.BigEndian.Uint32(data[off+keyLen : off+keyLen+4]) + e.ChildPage = binary.BigEndian.Uint32(data[off+keyLen+4 : off+keyLen+8]) + } else { + // Trailing child — no key/recNo, just child pointer + // In CDX, the rightmost child is the child of the last separator entry + // Actually, for the last entry we only need the child pointer + // The previous entry's child was the LEFT child; we need RIGHT child + // stored at the position after last key entry + copy(e.Key, data[off:off+keyLen]) + e.RecNo = binary.BigEndian.Uint32(data[off+keyLen : off+keyLen+4]) + e.ChildPage = binary.BigEndian.Uint32(data[off+keyLen+4 : off+keyLen+8]) } entries[i] = e off += entrySize @@ -278,8 +290,19 @@ func DecodeIntKeys(data []byte, nKeys int, keyLen int) []IntKeyEntry { // Index represents an open CDX index file. type Index struct { - file *os.File - tags []*Tag + file *os.File + tags []*Tag + mmapData []byte // mmap'd file for zero-copy reads +} + +// readAt reads len(buf) bytes at offset — from mmap or file fallback. +func (idx *Index) readAt(buf []byte, offset int64) error { + if idx.mmapData != nil && offset >= 0 && int(offset)+len(buf) <= len(idx.mmapData) { + copy(buf, idx.mmapData[offset:offset+int64(len(buf))]) + return nil + } + _, err := idx.file.ReadAt(buf, offset) + return err } // Tag represents one index tag within a CDX file. @@ -317,6 +340,14 @@ func OpenIndex(path string) (*Index, error) { idx := &Index{file: f} + // mmap for zero-copy reads + if fi, err2 := f.Stat(); err2 == nil && fi.Size() > 0 { + if data, err2 := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), + syscall.PROT_READ, syscall.MAP_SHARED); err2 == nil { + idx.mmapData = data + } + } + // Read compound header (structural root at offset 0) rootHdr, err := ReadTagHeader(f, 0) if err != nil { @@ -327,7 +358,7 @@ func OpenIndex(path string) (*Index, error) { // Parse compound tag directory from the structural root's B-tree // The structural index keys are 10-byte tag names, and each leaf entry // points to the tag header at a specific file offset. - tagEntries := readCompoundTagList(f, rootHdr) + tagEntries := readCompoundTagList(idx, rootHdr) for _, entry := range tagEntries { tagHdr, err := ReadTagHeader(f, entry.offset) @@ -363,6 +394,10 @@ func OpenIndex(path string) (*Index, error) { // Close closes the CDX file. func (idx *Index) Close() error { + if idx.mmapData != nil { + syscall.Munmap(idx.mmapData) + idx.mmapData = nil + } return idx.file.Close() } @@ -406,7 +441,7 @@ type tagDirEntry struct { // readCompoundTagList reads tag names and offsets from the structural root. // CDX compound header: root page is a B-tree of tag entries. // Each leaf key = 10-byte tag name, record number = page offset / 512. -func readCompoundTagList(f *os.File, rootHdr *TagHeader) []tagDirEntry { +func readCompoundTagList(idx *Index, rootHdr *TagHeader) []tagDirEntry { var entries []tagDirEntry if rootHdr.RootPtr == 0 { return entries @@ -414,7 +449,7 @@ func readCompoundTagList(f *os.File, rootHdr *TagHeader) []tagDirEntry { // Read the root page of the structural index pageData := make([]byte, 512) - _, err := f.ReadAt(pageData, int64(rootHdr.RootPtr)) + err := idx.readAt(pageData, int64(rootHdr.RootPtr)) if err != nil { return entries } @@ -431,7 +466,7 @@ func readCompoundTagList(f *os.File, rootHdr *TagHeader) []tagDirEntry { // If compound leaf decoding didn't find entries, scan for tag headers if len(entries) == 0 { - entries = scanCompoundLeaves(f, rootHdr) + entries = scanCompoundLeaves(idx, rootHdr) } return entries @@ -440,10 +475,10 @@ func readCompoundTagList(f *os.File, rootHdr *TagHeader) []tagDirEntry { // scanCompoundLeaves scans the CDX file for tag headers. // CDX tag headers are at 0x400 (1024) byte boundaries. // Each tag header is followed by a page with the key expression string. -func scanCompoundLeaves(f *os.File, rootHdr *TagHeader) []tagDirEntry { +func scanCompoundLeaves(idx *Index, rootHdr *TagHeader) []tagDirEntry { var entries []tagDirEntry - fileInfo, err := f.Stat() + fileInfo, err := idx.file.Stat() if err != nil { return entries } @@ -458,8 +493,8 @@ func scanCompoundLeaves(f *os.File, rootHdr *TagHeader) []tagDirEntry { // Tag headers are at 0x400 boundaries but NOT the compound root itself for off := int64(0x400); off < fileSize; off += 0x200 { buf := make([]byte, 0x400) - n, err := f.ReadAt(buf, off) - if err != nil || n < 0x200 { + err := idx.readAt(buf, off) + if err != nil { continue } @@ -482,7 +517,7 @@ func scanCompoundLeaves(f *os.File, rootHdr *TagHeader) []tagDirEntry { if keyExpr == "" { // Key expression might be in the next page (+0x200 from header) exprBuf := make([]byte, 256) - f.ReadAt(exprBuf, off+0x200) + idx.readAt(exprBuf, off+0x200) for i := 0; i < len(exprBuf) && exprBuf[i] != 0; i++ { keyExpr += string(exprBuf[i]) } @@ -539,7 +574,7 @@ func (t *Tag) Seek(searchKey []byte) (uint32, bool) { func (t *Tag) seekPage(pageOffset int64, searchKey []byte) (uint32, bool) { buf := make([]byte, PageLen) - if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil { + if err := t.index.readAt(buf, pageOffset); err != nil { t.tagEOF = true return 0, false } @@ -618,7 +653,7 @@ func (t *Tag) GoTop() bool { func (t *Tag) goLeftmost(pageOffset int64) bool { buf := make([]byte, PageLen) - if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil { + if err := t.index.readAt(buf, pageOffset); err != nil { return false } @@ -663,7 +698,7 @@ func (t *Tag) GoBottom() bool { func (t *Tag) goRightmost(pageOffset int64) bool { buf := make([]byte, PageLen) - if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil { + if err := t.index.readAt(buf, pageOffset); err != nil { return false } @@ -710,7 +745,7 @@ func (t *Tag) SkipNext() bool { keyIdx := t.stack[level].KeyIndex buf := make([]byte, PageLen) - if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil { + if err := t.index.readAt(buf, pageOffset); err != nil { t.tagEOF = true return false } @@ -730,7 +765,7 @@ func (t *Tag) SkipNext() bool { if hdr.RightPtr != 0 && hdr.RightPtr != 0xFFFFFFFF { nextOff := int64(hdr.RightPtr) buf2 := make([]byte, PageLen) - if _, err := t.index.file.ReadAt(buf2, nextOff); err != nil { + if err := t.index.readAt(buf2, nextOff); err != nil { t.tagEOF = true return false } @@ -761,7 +796,7 @@ func (t *Tag) SkipPrev() bool { keyIdx := t.stack[level].KeyIndex buf := make([]byte, PageLen) - if _, err := t.index.file.ReadAt(buf, pageOffset); err != nil { + if err := t.index.readAt(buf, pageOffset); err != nil { t.tagBOF = true return false } @@ -781,7 +816,7 @@ func (t *Tag) SkipPrev() bool { if hdr.LeftPtr != 0 && hdr.LeftPtr != 0xFFFFFFFF { prevOff := int64(hdr.LeftPtr) buf2 := make([]byte, PageLen) - if _, err := t.index.file.ReadAt(buf2, prevOff); err != nil { + if err := t.index.readAt(buf2, prevOff); err != nil { t.tagBOF = true return false }