// Copyright (c) 2026 Charles KWON OhJun (charleskwonohjun@gmail.com) // All rights reserved. // DBFArea — the core DBF file driver. // Byte-compatible with Harbour/Clipper DBF files. // // Harbour equivalent: DBFAREA in dbf1.c // Inherits from BaseArea (WAAREA), implements Area + RecordManager + Locker. // // Reference: // /mnt/d/harbour-core/src/rdd/dbf1.c // docs/dbf-engine-spec.md package dbf import ( "five/hbrt" "five/hbrdd" "fmt" "os" "strings" "syscall" ) // DBFArea implements the DBF database driver. // Harbour: struct DBFAREA in dbf1.c type DBFArea struct { hbrdd.BaseArea // embed WAAREA defaults // File handles dataFile *os.File filePath string shared bool readOnly bool // Header header Header fieldDescs []FieldDesc offsets []uint16 // field byte offsets within record // Record buffer — COW: recBuf may point into mmap (read-only) or ownBuf (writable) recBuf []byte // current record view (mmap slice or ownBuf) ownBuf []byte // owned writable buffer (allocated once) recNo uint32 // current record number (1-based) dirty bool // record buffer modified recOwned bool // true = recBuf is ownBuf (writable), false = mmap slice (read-only) // State recCount uint32 ghost bool // at phantom record (after APPEND) recLoaded bool // false = recBuf stale, need loadRecord() // Append batch buffer — accumulates records for single write at flush appendBuf []byte // buffered appended records (not yet written to disk) appendStart uint32 // first recNo in appendBuf (1-based) // mmap for zero-copy record reads mmapData []byte // Memo file (FPT) memoFile *FPTFile // Index integration (NTX/CDX) idxState *indexState // File locking state (byte-range locks via fcntl) fileLocked bool // FLOCK() held lockedRecs map[uint32]bool // records locked by DBRLOCK() // Field position cache — UPPER(name) → 1-based index. // Built lazily on first FieldPosCache() call. // SQLite: "column affinity binding" — O(1) vs O(n) linear scan. fieldPosMap map[string]int } // DBFDriver is the driver factory for DBF files. type DBFDriver struct{} func (d *DBFDriver) Name() string { return "DBF" } func (d *DBFDriver) Open(params hbrdd.OpenParams) (hbrdd.Area, error) { return openDBF(d, params) } func (d *DBFDriver) Create(params hbrdd.CreateParams) (hbrdd.Area, error) { return createDBF(d, params) } func init() { hbrdd.RegisterDriver(&DBFDriver{}) // Register aliases used by Harbour hbrdd.RegisterDriver(&dbfAliasDriver{name: "DBFNTX"}) hbrdd.RegisterDriver(&dbfAliasDriver{name: "DBFCDX"}) hbrdd.RegisterDriver(&dbfAliasDriver{name: "DBFFPT"}) // SIX compatible drivers — same DBF engine, different semantics handled at higher level hbrdd.RegisterDriver(&dbfAliasDriver{name: "SIXCDX"}) hbrdd.RegisterDriver(&dbfAliasDriver{name: "DBFNSX"}) hbrdd.RegisterDriver(&dbfAliasDriver{name: "DBFSIX"}) // Transfer format aliases hbrdd.RegisterDriver(&dbfAliasDriver{name: "DBFDBT"}) // Bitmap/Rushmore RDD variants hbrdd.RegisterDriver(&dbfAliasDriver{name: "BMDBFNTX"}) hbrdd.RegisterDriver(&dbfAliasDriver{name: "BMDBFCDX"}) hbrdd.RegisterDriver(&dbfAliasDriver{name: "BMDBFNSX"}) } // dbfAliasDriver wraps DBFDriver with a different name. type dbfAliasDriver struct { name string } func (d *dbfAliasDriver) Name() string { return d.name } func (d *dbfAliasDriver) Open(params hbrdd.OpenParams) (hbrdd.Area, error) { return openDBF(&DBFDriver{}, params) } func (d *dbfAliasDriver) Create(params hbrdd.CreateParams) (hbrdd.Area, error) { return createDBF(&DBFDriver{}, params) } // fptPathFromDBF returns the FPT memo file path for a given DBF path. func fptPathFromDBF(dbfPath string) string { base := dbfPath if idx := strings.LastIndex(base, "."); idx >= 0 { base = base[:idx] } return base + ".fpt" } // hasMemoField checks if any field descriptor is a MEMO type. func hasMemoField(fields []FieldDesc) bool { for _, f := range fields { if f.Type == 'M' || f.Type == 'm' { return true } } return false } // --- Open --- // Harbour: hb_dbfOpen in dbf1.c func openDBF(drv *DBFDriver, params hbrdd.OpenParams) (*DBFArea, error) { path := params.Path if !hasExtension(path) { path += ".dbf" } flag := os.O_RDWR if params.ReadOnly { flag = os.O_RDONLY } f, err := os.OpenFile(path, flag, 0) if err != nil { return nil, fmt.Errorf("open %s: %w", path, err) } area := &DBFArea{ dataFile: f, filePath: path, shared: params.Shared, readOnly: params.ReadOnly, } area.BaseArea = hbrdd.BaseArea{} // Step 1: Read header (32 bytes) hdr, err := ReadHeader(f) if err != nil { f.Close() return nil, err } area.header = *hdr // Step 2: Read field descriptors fieldCount := hdr.FieldCount() if fieldCount <= 0 { f.Close() return nil, fmt.Errorf("invalid field count: %d", fieldCount) } fields, err := ReadFieldDescs(f, fieldCount) if err != nil { f.Close() return nil, err } area.fieldDescs = fields // Step 3: Build field offsets area.offsets = BuildFieldOffsets(fields) // Step 4: Validate record length expectedRecLen := area.offsets[len(fields)] if uint16(expectedRecLen) != hdr.RecordLen { // Allow minor discrepancy (some DBF writers add extra bytes) if uint16(expectedRecLen) > hdr.RecordLen { f.Close() return nil, fmt.Errorf("field offsets (%d) exceed record length (%d)", expectedRecLen, hdr.RecordLen) } } // Step 5: Allocate record buffer area.ownBuf = make([]byte, hdr.RecordLen) area.recBuf = area.ownBuf area.recOwned = true // Step 6: Set record count (shared mode: recalculate from file size) if params.Shared { fileSize, _ := f.Seek(0, 2) area.recCount = uint32((fileSize - int64(hdr.HeaderLen)) / int64(hdr.RecordLen)) } else { area.recCount = hdr.RecCount } // Step 7: Build FieldInfo for BaseArea fieldInfos := make([]hbrdd.FieldInfo, fieldCount) for i, fd := range fields { fieldInfos[i] = hbrdd.FieldInfo{ Name: fd.GetName(), Type: fd.Type, Len: int(fd.Len), Dec: int(fd.Dec), Flags: fd.Flags, } } area.InitFields(fieldInfos) // Step 8: Auto-open FPT if memo fields exist if hasMemoField(fields) { fptPath := fptPathFromDBF(path) if fpt, err := OpenFPT(fptPath); err == nil { area.memoFile = fpt } // If FPT doesn't exist, memo reads return empty string } // Step 9: mmap DBF for zero-copy record reads area.mmapDBF() // Step 10: Position at first record area.FEof = (area.recCount == 0) if area.recCount > 0 { area.GoTo(1) } return area, nil } // --- Create --- func createDBF(drv *DBFDriver, params hbrdd.CreateParams) (*DBFArea, error) { path := params.Path if !hasExtension(path) { path += ".dbf" } f, err := os.Create(path) if err != nil { return nil, fmt.Errorf("create %s: %w", path, err) } // Build field descriptors fieldDescs := make([]FieldDesc, len(params.Fields)) recordLen := uint16(1) // deletion flag for i, fi := range params.Fields { fieldDescs[i].SetName(fi.Name) fieldDescs[i].Type = fi.Type fieldDescs[i].Len = byte(fi.Len) fieldDescs[i].Dec = byte(fi.Dec) fieldDescs[i].Flags = fi.Flags recordLen += uint16(fi.Len) } // Build header hasMemo := hasMemoField(fieldDescs) headerLen := uint16(HeaderSize + len(fieldDescs)*FieldDescSize + 1) // +1 for terminator version := byte(VersionDBF3) if hasMemo { version = VersionFPT } hdr := Header{ Version: version, RecCount: 0, HeaderLen: headerLen, RecordLen: recordLen, } hdr.UpdateDate() // Write header if err := WriteHeader(f, &hdr); err != nil { f.Close() return nil, err } // Write field descriptors if err := WriteFieldDescs(f, fieldDescs); err != nil { f.Close() return nil, err } // Write header terminator f.Write([]byte{HeaderTerminator}) // Write EOF marker f.Write([]byte{EOFMarker}) f.Seek(0, 0) area := &DBFArea{ dataFile: f, filePath: path, header: hdr, fieldDescs: fieldDescs, offsets: BuildFieldOffsets(fieldDescs), ownBuf: make([]byte, recordLen), recOwned: true, recCount: 0, } area.recBuf = area.ownBuf fieldInfos := make([]hbrdd.FieldInfo, len(params.Fields)) copy(fieldInfos, params.Fields) area.InitFields(fieldInfos) area.FEof = true // Auto-create FPT if memo fields exist if hasMemo { fptPath := fptPathFromDBF(path) fpt, err := CreateFPT(fptPath, FPTDefaultBlock) if err != nil { f.Close() return nil, fmt.Errorf("create memo file: %w", err) } area.memoFile = fpt } return area, nil } // --- Area interface --- func (a *DBFArea) Driver() hbrdd.Driver { return &DBFDriver{} } func (a *DBFArea) Close() error { a.unmapDBF() // unmap before writing if a.dirty { a.flushRecord() } a.dataFile.WriteAt([]byte{EOFMarker}, a.header.EOFOffset()) a.updateHeader() // Release any held byte-range locks before closing the fd — POSIX // drops them implicitly on close, but being explicit avoids races // with other workareas sharing the same underlying file. a.releaseAllLocks() if a.memoFile != nil { a.memoFile.Close() a.memoFile = nil } err := a.dataFile.Close() a.BaseArea.Close() return err } // MemoFile returns the FPT memo file, or nil if no memo fields. func (a *DBFArea) MemoFile() *FPTFile { return a.memoFile } // FullPath returns the on-disk path of the DBF. For dbInfo(DBI_FULLPATH). func (a *DBFArea) FullPath() string { return a.filePath } // IsShared returns true if the area was opened shared. // For dbInfo(DBI_SHARED). func (a *DBFArea) IsShared() bool { return a.shared } // IsReadOnly returns true if the area was opened read-only. // For dbInfo(DBI_ISREADONLY). func (a *DBFArea) IsReadOnly() bool { return a.readOnly } // FieldPosCache returns the 1-based field position for a field name. // Uses a lazily-built hash map for O(1) lookup instead of O(n) linear scan. // SQLite: "column affinity binding" — critical for SQL engines that call // FieldPos() hundreds of thousands of times per query. func (a *DBFArea) FieldPosCache(name string) int { if a.fieldPosMap == nil { a.fieldPosMap = make(map[string]int, len(a.fieldDescs)) for i, fd := range a.fieldDescs { // Trim null bytes + spaces from the [11]byte field name n := 0 for n < 11 && fd.Name[n] != 0 { n++ } fname := strings.ToUpper(strings.TrimSpace(string(fd.Name[:n]))) a.fieldPosMap[fname] = i + 1 } } if pos, ok := a.fieldPosMap[name]; ok { return pos } return 0 } func (a *DBFArea) Flush() error { if a.dirty { if err := a.flushRecord(); err != nil { return err } } a.unmapDBF() a.dataFile.WriteAt([]byte{EOFMarker}, a.header.EOFOffset()) a.updateHeader() err := a.dataFile.Sync() a.mmapDBF() // re-mmap after flush return err } // --- Movement --- func (a *DBFArea) RecNo() uint32 { return a.recNo } func (a *DBFArea) RecCount() (uint32, error) { if a.shared { // Recalculate from file size (Harbour behavior) size, err := a.dataFile.Seek(0, 2) if err != nil { return a.recCount, err } a.recCount = uint32((size - int64(a.header.HeaderLen)) / int64(a.header.RecordLen)) } return a.recCount, nil } func (a *DBFArea) Deleted() bool { a.loadRecord() if len(a.recBuf) > 0 { return a.recBuf[0] == RecordDeleted } return false } // GoTo positions the cursor at a specific record number. // Harbour: hb_dbfGoTo in dbf1.c — lazy read: record loaded on first access. func (a *DBFArea) GoTo(recNo uint32) error { if a.dirty { a.flushRecord() } a.FFound = false if recNo == 0 || recNo > a.recCount { a.recNo = a.recCount + 1 a.FEof = true a.FBof = (recNo == 0) a.recLoaded = false a.recBuf = a.ownBuf a.recOwned = true for i := range a.recBuf { a.recBuf[i] = ' ' } return nil } // Read record — COW: mmap slice reference (zero-copy), fallback to file read offset := a.header.RecordOffset(recNo) recLen := int(a.header.RecordLen) if a.mmapData != nil && int(offset)+recLen <= len(a.mmapData) { // Zero-copy: recBuf points into mmap (read-only until PutValue) a.recBuf = a.mmapData[offset : offset+int64(recLen)] a.recOwned = false } else if _, err := a.dataFile.ReadAt(a.ownBuf, offset); err != nil { a.FEof = true return fmt.Errorf("read record %d: %w", recNo, err) } else { a.recBuf = a.ownBuf a.recOwned = true } a.recNo = recNo a.recLoaded = true a.FEof = false a.FBof = false a.dirty = false a.ghost = false return nil } // GoTop positions at the first record. // Harbour: hb_dbfGoTop func (a *DBFArea) GoTop() error { // Use index order if active if a.idxState != nil && a.idxState.current >= 0 { a.FTop = true a.FBottom = false return a.GoTopIndexed() } a.FTop = true a.FBottom = false if a.recCount == 0 { a.FEof = true a.recNo = 1 return nil } if err := a.GoTo(1); err != nil { return err } // Skip filtered/deleted records return a.skipFilter(1) } // GoBottom positions at the last record. // Harbour: hb_dbfGoBottom func (a *DBFArea) GoBottom() error { // Use index order if active if a.idxState != nil && a.idxState.current >= 0 { a.FTop = false a.FBottom = true return a.GoBottomIndexed() } a.FTop = false a.FBottom = true if a.recCount == 0 { a.FEof = true a.recNo = 1 return nil } if err := a.GoTo(a.recCount); err != nil { return err } return a.skipFilter(-1) } // Skip moves the cursor by count records. // Harbour: hb_dbfSkip func (a *DBFArea) Skip(count int64) error { // Use index order if active if a.idxState != nil && a.idxState.current >= 0 { a.FTop = false a.FBottom = false return a.SkipIndexed(count) } a.FTop = false a.FBottom = false if count == 0 { // Skip 0 = re-evaluate filter at current position return a.skipFilter(1) } if count > 0 { for i := int64(0); i < count; i++ { if a.FEof { break } newRec := a.recNo + 1 if newRec > a.recCount { // Flush dirty record before entering EOF phantom if a.dirty { a.flushRecord() } a.FEof = true a.recNo = a.recCount + 1 break } if err := a.GoTo(newRec); err != nil { return err } // Skip deleted records when SET DELETED ON if err := a.skipFilter(1); err != nil { return err } } } else { for i := int64(0); i > count; i-- { if a.recNo <= 1 { a.FBof = true break } if err := a.GoTo(a.recNo - 1); err != nil { return err } if err := a.skipFilter(-1); err != nil { return err } } } return nil } // mmapDBF maps the DBF file for zero-copy reads. Called after open. func (a *DBFArea) mmapDBF() { fi, err := a.dataFile.Stat() if err != nil || fi.Size() < int64(a.header.HeaderLen) { return } data, err := syscall.Mmap(int(a.dataFile.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) if err != nil { return } a.mmapData = data } // unmapDBF releases the mmap. func (a *DBFArea) unmapDBF() { if a.mmapData != nil { syscall.Munmap(a.mmapData) a.mmapData = nil } } // loadRecord reads the current record — from mmap or file fallback. func (a *DBFArea) loadRecord() { if a.recLoaded || a.FEof || a.recNo == 0 || a.recNo > a.recCount { return } offset := a.header.RecordOffset(a.recNo) a.dataFile.ReadAt(a.recBuf, offset) a.recLoaded = true } // --- Field access --- func (a *DBFArea) GetValue(fieldIndex int) (hbrt.Value, error) { a.loadRecord() if fieldIndex < 0 || fieldIndex >= len(a.fieldDescs) { return hbrt.MakeNil(), fmt.Errorf("field index out of range: %d", fieldIndex) } if a.FEof { return hbrt.MakeNil(), nil } fd := &a.fieldDescs[fieldIndex] // MEMO field: read from FPT and return string if (fd.Type == 'M' || fd.Type == 'm') && a.memoFile != nil { blockVal := GetFieldValue(a.recBuf, a.offsets[fieldIndex], fd) blockNo := uint32(blockVal.AsNumInt()) if blockNo == 0 { return hbrt.MakeString(""), nil } data, err := a.memoFile.ReadMemo(blockNo) if err != nil { return hbrt.MakeString(""), nil } return hbrt.MakeString(string(data)), nil } return GetFieldValue(a.recBuf, a.offsets[fieldIndex], fd), nil } func (a *DBFArea) PutValue(fieldIndex int, val hbrt.Value) error { a.loadRecord() // COW: promote read-only mmap slice to writable owned buffer if !a.recOwned { copy(a.ownBuf, a.recBuf) a.recBuf = a.ownBuf a.recOwned = true } if a.readOnly { return fmt.Errorf("table is read-only") } if fieldIndex < 0 || fieldIndex >= len(a.fieldDescs) { return fmt.Errorf("field index out of range: %d", fieldIndex) } fd := &a.fieldDescs[fieldIndex] // MEMO field: write string to FPT, store block number in DBF if (fd.Type == 'M' || fd.Type == 'm') && a.memoFile != nil && val.IsString() { data := []byte(val.AsString()) blockNo, err := a.memoFile.WriteMemo(data) if err != nil { return fmt.Errorf("write memo: %w", err) } putMemoRef(a.recBuf[a.offsets[fieldIndex]:a.offsets[fieldIndex]+uint16(fd.Len)], fd.Len, blockNo) a.dirty = true return nil } PutFieldValue(a.recBuf, a.offsets[fieldIndex], fd, val) a.dirty = true return nil } // --- Record operations --- // Append adds a new blank record. // Harbour: hb_dbfAppend — writes are deferred until Flush/Close/GoTo. func (a *DBFArea) Append() error { if a.readOnly { return fmt.Errorf("table is read-only") } // Batch consecutive APPENDs: save current dirty record to appendBuf instead of writing to disk. if a.dirty && a.ghost { // Previous was also an APPEND — accumulate in batch buffer (no syscall) recLen := int(a.header.RecordLen) if a.appendBuf == nil { a.appendStart = a.recNo a.appendBuf = make([]byte, 0, recLen*256) // pre-alloc for ~256 records } a.appendBuf = append(a.appendBuf, a.recBuf[:recLen]...) } else if a.dirty { a.flushRecord() } // Unmap — file will grow if a.mmapData != nil { a.unmapDBF() } a.recCount++ a.recNo = a.recCount a.header.RecCount = a.recCount // Promote to owned buffer for writing a.recBuf = a.ownBuf a.recOwned = true for i := range a.recBuf { a.recBuf[i] = ' ' } a.FEof = false a.FBof = false a.dirty = true a.ghost = true a.recLoaded = true return nil } // Delete marks the current record as deleted. func (a *DBFArea) Delete() error { if a.readOnly || a.FEof { return nil } if !a.recOwned { copy(a.ownBuf, a.recBuf) a.recBuf = a.ownBuf a.recOwned = true } a.recBuf[0] = RecordDeleted a.dirty = true return nil } // Recall undeletes the current record. func (a *DBFArea) Recall() error { if a.readOnly || a.FEof { return nil } if !a.recOwned { copy(a.ownBuf, a.recBuf) a.recBuf = a.ownBuf a.recOwned = true } a.recBuf[0] = RecordActive a.dirty = true return nil } // Pack removes all deleted records. // Harbour: hb_dbfPack — requires exclusive access. func (a *DBFArea) Pack() error { if a.readOnly { return fmt.Errorf("table is read-only") } if a.shared { return fmt.Errorf("PACK requires exclusive access") } if a.dirty { a.flushRecord() } // Temporarily disable index to avoid indexed navigation during PACK var savedIdx *indexState if a.idxState != nil { savedIdx = a.idxState a.idxState = nil } outRec := uint32(0) buf := make([]byte, a.header.RecordLen) for recNo := uint32(1); recNo <= a.recCount; recNo++ { offset := a.header.RecordOffset(recNo) if _, err := a.dataFile.ReadAt(buf, offset); err != nil { return err } if buf[0] != RecordDeleted { outRec++ outOffset := a.header.RecordOffset(outRec) if _, err := a.dataFile.WriteAt(buf, outOffset); err != nil { return err } } } a.recCount = outRec a.header.RecCount = outRec // Truncate file newSize := a.header.EOFOffset() + 1 // +1 for EOF marker a.dataFile.Truncate(newSize) // Write EOF a.dataFile.WriteAt([]byte{EOFMarker}, a.header.EOFOffset()) // Update header a.updateHeader() // Reposition (natural order, no index yet) if a.recCount > 0 { a.GoTo(1) } else { a.FEof = true a.recNo = 1 } // Rebuild indexes (record numbers changed after PACK) if savedIdx != nil { a.idxState = savedIdx if err := a.OrderListRebuild(); err != nil { return err } } return nil } // Zap removes all records. func (a *DBFArea) Zap() error { if a.readOnly || a.shared { return fmt.Errorf("ZAP requires exclusive access") } // Save index state var savedIdx *indexState if a.idxState != nil { savedIdx = a.idxState a.idxState = nil } a.recCount = 0 a.header.RecCount = 0 // Truncate to header + EOF a.dataFile.Truncate(int64(a.header.HeaderLen) + 1) a.dataFile.WriteAt([]byte{EOFMarker}, int64(a.header.HeaderLen)) a.updateHeader() a.FEof = true a.recNo = 1 // Rebuild indexes (empty after ZAP) if savedIdx != nil { a.idxState = savedIdx a.OrderListRebuild() // rebuilds empty indexes } return nil } // --- Internal --- func (a *DBFArea) flushRecord() error { if !a.dirty || a.FEof { return nil } // Flush any accumulated append batch first (single write for all buffered records) if len(a.appendBuf) > 0 { offset := a.header.RecordOffset(a.appendStart) recLen := int(a.header.RecordLen) // Append current dirty record to batch too all := append(a.appendBuf, a.recBuf[:recLen]...) _, err := a.dataFile.WriteAt(all, offset) a.appendBuf = nil a.appendStart = 0 if err == nil { a.dirty = false } return err } offset := a.header.RecordOffset(a.recNo) _, err := a.dataFile.WriteAt(a.recBuf, offset) if err == nil { a.dirty = false } return err } func (a *DBFArea) updateHeader() { a.header.RecCount = a.recCount a.header.UpdateDate() a.dataFile.Seek(0, 0) WriteHeader(a.dataFile, &a.header) } // skipFilter advances cursor past deleted/filtered records. // Harbour: hb_dbfSkipFilter in dbf1.c // direction: 1 = forward, -1 = backward func (a *DBFArea) skipFilter(direction int64) error { if direction == 0 { direction = 1 } setDel := hbrdd.IsSetDeleted != nil && hbrdd.IsSetDeleted() if !setDel { return nil } for !a.FEof && !a.FBof { if !a.Deleted() { break } // Move to next/prev record if direction > 0 { newRec := a.recNo + 1 if newRec > a.recCount { a.FEof = true a.recNo = a.recCount + 1 break } if err := a.GoTo(newRec); err != nil { return err } } else { if a.recNo <= 1 { a.FBof = true if err := a.GoTo(1); err != nil { return err } break } if err := a.GoTo(a.recNo - 1); err != nil { return err } } } return nil } // --- Helpers --- func hasExtension(path string) bool { for i := len(path) - 1; i >= 0; i-- { if path[i] == '.' { return true } if path[i] == '/' || path[i] == '\\' { return false } } return false }