From dfc2f42e798d488e1efc9937935436f984f7fe3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Czerpak?= Date: Fri, 17 Oct 2014 14:55:16 +0200 Subject: [PATCH] 2014-10-17 14:55 UTC+0200 Przemyslaw Czerpak (druzus/at/poczta.onet.pl) * include/hbrddcdx.h * src/rdd/dbfcdx/dbfcdx1.c + added support for large index files over 4GB length. These are slightly modified CDX indexes which stores index page numbers instead of index page offsets inside index file. This trick increase maximum index files size from 2^32 (4GB) to 2^41 (2TB). This index format is enabled automatically when DB_DBFLOCK_HB64 is used. This is the same behavior as in DBFNTX and DBFNSX for which I added support for large indexes (up to 4TB) few years ago. Warning: new CDX indexes are not backward compatible and cannot be read by other systems or older [x]Harbour versions. If you try to open new indexes using older [x]Harbour RDDs then RTE "DBFCDX/1012 Corruption detected" is generated. When current Harbour *DBFCDX/SIXCDX RDD open index file then it automatically recognize type of index file so it will work correctly with both versions without any problem. In short words: People using DB_DBFLOCK_HB64 should remember that after reindexing with new Harbour applications old ones cannot read new CDX indexes. ; In next step I plan to add support for user defined page size in CDX index files. * doc/xhb-diff.txt * added information about extended CDX format to section "NATIVE RDDs" * src/rdd/dbfcdx/dbfcdx1.c * src/rdd/dbfnsx/dbfnsx1.c * src/rdd/dbfntx/dbfntx1.c * disable record readahead buffer used during indexing when only one record can be stored inside ! generate RTE when data cannot be read into record readahead buffer during indexing --- ChangeLog.txt | 34 ++++++++ doc/xhb-diff.txt | 24 +++--- include/hbrddcdx.h | 13 +-- src/rdd/dbfcdx/dbfcdx1.c | 167 +++++++++++++++++++++++++-------------- src/rdd/dbfnsx/dbfnsx1.c | 25 +++--- src/rdd/dbfntx/dbfntx1.c | 25 +++--- 6 files changed, 196 insertions(+), 92 deletions(-) diff --git a/ChangeLog.txt b/ChangeLog.txt index 19fe10bf36..6730f30b95 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -10,6 +10,40 @@ * Change, ! Fix, % Optimization, + Addition, - Removal, ; Comment */ +2014-10-17 14:55 UTC+0200 Przemyslaw Czerpak (druzus/at/poczta.onet.pl) + * include/hbrddcdx.h + * src/rdd/dbfcdx/dbfcdx1.c + + added support for large index files over 4GB length. + These are slightly modified CDX indexes which stores index page numbers + instead of index page offsets inside index file. This trick increase + maximum index files size from 2^32 (4GB) to 2^41 (2TB). This index + format is enabled automatically when DB_DBFLOCK_HB64 is used. This is + the same behavior as in DBFNTX and DBFNSX for which I added support + for large indexes (up to 4TB) few years ago. + Warning: new CDX indexes are not backward compatible and cannot be + read by other systems or older [x]Harbour versions. + If you try to open new indexes using older [x]Harbour RDDs + then RTE "DBFCDX/1012 Corruption detected" is generated. + When current Harbour *DBFCDX/SIXCDX RDD open index file + then it automatically recognize type of index file so it + will work correctly with both versions without any problem. + In short words: People using DB_DBFLOCK_HB64 should remember + that after reindexing with new Harbour applications old ones + cannot read new CDX indexes. + ; In next step I plan to add support for user defined page size in CDX + index files. + + * doc/xhb-diff.txt + * added information about extended CDX format to section "NATIVE RDDs" + + * src/rdd/dbfcdx/dbfcdx1.c + * src/rdd/dbfnsx/dbfnsx1.c + * src/rdd/dbfntx/dbfntx1.c + * disable record readahead buffer used during indexing when only + one record can be stored inside + ! generate RTE when data cannot be read into record readahead buffer + during indexing + 2014-10-14 14:05 UTC+0200 Przemyslaw Czerpak (druzus/at/poczta.onet.pl) * src/rtl/gtxwc/gtxwc.c ! fixed size of picture shown by HB_GTI_DISPIMAGE diff --git a/doc/xhb-diff.txt b/doc/xhb-diff.txt index c179c3b7ad..2679184158 100644 --- a/doc/xhb-diff.txt +++ b/doc/xhb-diff.txt @@ -2334,7 +2334,7 @@ or SMT and size of memo block. It's limited by maximal number of memo blocks = 2^32 and size of memo block so it's 2^32*. The default memo block size for DBT is 512 bytes, FPT - 64 bytes and for SMT 32 bytes. So for standard memo block sizes the maximum are: -DBT->2TB, FPT->256GB, SMT->128GB. The maximal memo block size in +DBT->l2TB, FPT->256GB, SMT->128GB. The maximal memo block size in Harbour is 2^32 and minimal is 1 byte and it can be any value between 1 and 65536 and then any number of 64KB blocks. The last limitation is introduced as workaround for some wrongly implemented in other @@ -2356,14 +2356,20 @@ extensions are supported by NTX in [x]Harbour. The NSX format in [x]Harbour is also limited by default to 4GB but like in NTX enabling 64bit locking extend it to 4TB. It also supports common to NTX and CDX set of features. - -The CDX format is limited to 4GB and so far [x]Harbour does not support -extended mode which can increase the size up to 2TB with standard page -length and it can be bigger in all formats if we introduce support for -bigger index pages. Of course all such extended formats are not binary -compatible with original ones and so far can be used only by [x]Harbour -RDDs though in ADS the .adi format is such extended CDX format so maybe -in the future it will be possible to use .adi indexes in our CDX RDD. +The CDX format is limited to 4GB by it's internal structure. In Harbour +just like in NTX and NSX enabling 64bit locking slightly change the +format extending increasing maximum index size to 2TB for standard +CDX page length (512 bytes). So far xHarbour does not support this +extended format in DBFCDX so it cannot read such indexes created by +Harbour. +When extended formats are used then above limits can be bigger in all +RDDs if we introduce support for bigger index pages. I plan to add it +in the future. +Of course all such extended formats are not binary compatible with +original ones and so far can be used only by Harbour and xHarbour +(except DBFCDX) RDDs though in ADS the .adi format is such extended +CDX format so maybe in the future it will be possible to use .adi +indexes in our CDX RDD. Of course all of the above sizes can be reduced by operating system (OS) or file system (FS) limitations so it's necessary to check what is diff --git a/include/hbrddcdx.h b/include/hbrddcdx.h index cb5e6a843a..c0dfcc48c0 100644 --- a/include/hbrddcdx.h +++ b/include/hbrddcdx.h @@ -58,7 +58,8 @@ HB_EXTERN_BEGIN #define CDX_MAXKEY 240 #define CDX_MAXEXP 255 #define CDX_MAXTAGNAMELEN 10 -#define CDX_PAGELEN 512 +#define CDX_PAGELEN_BITS 9 +#define CDX_PAGELEN (1<fLargeFile ? CDX_PAGELEN_BITS : 0 ) ) +#define hb_cdxFilePageNum( I, O ) ( ( HB_ULONG ) ( ( O ) >> ( ( I )->fLargeFile ? CDX_PAGELEN_BITS : 0 ) ) ) +#define hb_cdxFilePageNext( I, C ) ( ( C ) << ( ( I )->fLargeFile ? 0 : CDX_PAGELEN_BITS ) ) +#define hb_cdxFilePageRootValid( I, B ) ( ( I )->fLargeFile ? CDX_PAGELEN != CDX_DUMMYNODE : ( ( B ) % CDX_PAGELEN == 0 ) ) +#else +#define hb_cdxFilePageOffset( I, B ) ( ( HB_FOFFSET ) ( B ) ) +#define hb_cdxFilePageNum( I, O ) ( ( HB_ULONG ) ( O ) ) +#define hb_cdxFilePageNext( I, C ) ( ( C ) << CDX_PAGELEN_BITS ) +#define hb_cdxFilePageRootValid( I, B ) ( ( B ) % CDX_PAGELEN == 0 ) +#endif /* * Tag->fRePos = HB_TRUE means that rootPage->...->childLeafPage path is * bad and has to be reloaded @@ -994,11 +1005,11 @@ static void hb_cdxIndexLockFlush( LPCDXINDEX pIndex ) /* * get free index page */ -static HB_ULONG hb_cdxIndexGetAvailPage( LPCDXINDEX pIndex, HB_BOOL bHeader ) +static HB_ULONG hb_cdxIndexGetAvailPage( LPCDXINDEX pIndex, HB_BOOL fHeader ) { PHB_FILE pFile = pIndex->pFile; HB_BYTE byBuf[ 4 ]; - HB_ULONG ulPos; + HB_ULONG ulPage; if( pIndex->fReadonly ) { @@ -1009,19 +1020,19 @@ static HB_ULONG hb_cdxIndexGetAvailPage( LPCDXINDEX pIndex, HB_BOOL bHeader ) hb_errInternal( 9102, "hb_cdxIndexGetAvailPage on not locked index file.", NULL, NULL ); } - if( pIndex->freePage != 0 && pIndex->freePage != CDX_DUMMYNODE && ! bHeader ) + if( pIndex->freePage != 0 && pIndex->freePage != CDX_DUMMYNODE && ! fHeader ) { - ulPos = pIndex->freePage; + ulPage = pIndex->freePage; if( pIndex->freeLst != NULL ) { LPCDXLIST pLst = pIndex->freeLst; - pIndex->freePage = pLst->ulAddr; + pIndex->freePage = pLst->nextPage; pIndex->freeLst = pLst->pNext; hb_xfree( pLst ); } else { - if( hb_fileReadAt( pFile, byBuf, 4, ulPos ) != 4 ) + if( hb_fileReadAt( pFile, byBuf, 4, hb_cdxFilePageOffset( pIndex, ulPage ) ) != 4 ) hb_errInternal( EDBF_READ, "hb_cdxIndexGetAvailPage: Read index page failed.", NULL, NULL ); pIndex->freePage = HB_GET_LE_UINT32( byBuf ); #ifdef HB_CDX_DBGUPDT @@ -1031,19 +1042,19 @@ static HB_ULONG hb_cdxIndexGetAvailPage( LPCDXINDEX pIndex, HB_BOOL bHeader ) } else { - int iCnt = ( bHeader ? CDX_HEADERPAGES : 1 ); + int iCnt = ( fHeader ? CDX_HEADERPAGES : 1 ); - if( pIndex->nextAvail != CDX_DUMMYNODE ) - ulPos = pIndex->nextAvail; - else - ulPos = ( HB_ULONG ) hb_fileSize( pFile ); - pIndex->nextAvail = ulPos + iCnt * CDX_PAGELEN; + if( pIndex->nextAvail == CDX_DUMMYNODE ) + pIndex->nextAvail = hb_cdxFilePageNum( pIndex, hb_fileSize( pFile ) ); + + ulPage = pIndex->nextAvail; + pIndex->nextAvail += hb_cdxFilePageNext( pIndex, iCnt ); /* TODO: ### */ - if( bHeader ) + if( fHeader ) { HB_BYTE byPageBuf[ CDX_PAGELEN ]; - HB_FOFFSET fOffset = ulPos; + HB_FOFFSET fOffset = hb_cdxFilePageOffset( pIndex, ulPage ); hb_cdxIndexLockFlush( pIndex ); memset( byPageBuf, 0, CDX_PAGELEN ); @@ -1057,17 +1068,17 @@ static HB_ULONG hb_cdxIndexGetAvailPage( LPCDXINDEX pIndex, HB_BOOL bHeader ) pIndex->fChanged = HB_TRUE; } } - return ulPos; + return ulPage; } /* * free index page */ -static void hb_cdxIndexPutAvailPage( LPCDXINDEX pIndex, HB_ULONG ulPos, HB_BOOL bHeader ) +static void hb_cdxIndexPutAvailPage( LPCDXINDEX pIndex, HB_ULONG ulPage, HB_BOOL fHeader ) { - if( ulPos != 0 && ulPos != CDX_DUMMYNODE ) + if( ulPage != 0 && ulPage != CDX_DUMMYNODE ) { - int iCnt = ( bHeader ? CDX_HEADERPAGES : 1 ); + int iCnt = ( fHeader ? CDX_HEADERPAGES : 1 ); LPCDXLIST pLst; if( pIndex->fReadonly ) @@ -1075,16 +1086,17 @@ static void hb_cdxIndexPutAvailPage( LPCDXINDEX pIndex, HB_ULONG ulPos, HB_BOOL if( pIndex->fShared && ! pIndex->lockWrite ) hb_errInternal( 9102, "hb_cdxIndexPutAvailPage on not locked index file.", NULL, NULL ); - while( iCnt-- ) + do { pLst = ( LPCDXLIST ) hb_xgrab( sizeof( CDXLIST ) ); - pLst->ulAddr = pIndex->freePage; - pIndex->freePage = ulPos; + pLst->nextPage = pIndex->freePage; + pIndex->freePage = ulPage; pLst->fStat = HB_TRUE; pLst->pNext = pIndex->freeLst; pIndex->freeLst = pLst; - ulPos += CDX_PAGELEN; + ulPage += hb_cdxFilePageNext( pIndex, 1 ); } + while( --iCnt ); } } @@ -1095,16 +1107,16 @@ static void hb_cdxIndexFlushAvailPage( LPCDXINDEX pIndex ) { LPCDXLIST pLst = pIndex->freeLst; HB_BYTE byPageBuf[ CDX_PAGELEN ]; - HB_ULONG ulPos; + HB_ULONG ulPage; HB_BOOL fClean = HB_TRUE; if( pIndex->fReadonly ) - hb_errInternal( 9101, "hb_cdxIndexPutAvailPage on readonly database.", NULL, NULL ); + hb_errInternal( 9101, "hb_cdxIndexFlushAvailPage on readonly database.", NULL, NULL ); if( pIndex->fShared && ! pIndex->lockWrite ) - hb_errInternal( 9102, "hb_cdxIndexPutAvailPage on not locked index file.", NULL, NULL ); + hb_errInternal( 9102, "hb_cdxIndexFlushAvailPage on not locked index file.", NULL, NULL ); hb_cdxIndexLockFlush( pIndex ); - ulPos = pIndex->freePage; + ulPage = pIndex->freePage; while( pLst && pLst->fStat ) { if( fClean ) @@ -1112,13 +1124,12 @@ static void hb_cdxIndexFlushAvailPage( LPCDXINDEX pIndex ) memset( byPageBuf, 0, CDX_PAGELEN ); fClean = HB_FALSE; } - HB_PUT_LE_UINT32( byPageBuf, pLst->ulAddr ); - if( hb_fileWriteAt( pIndex->pFile, byPageBuf, CDX_PAGELEN, ulPos ) != CDX_PAGELEN ) - { + HB_PUT_LE_UINT32( byPageBuf, pLst->nextPage ); + if( hb_fileWriteAt( pIndex->pFile, byPageBuf, CDX_PAGELEN, + hb_cdxFilePageOffset( pIndex, ulPage ) ) != CDX_PAGELEN ) hb_errInternal( EDBF_WRITE, "Write in index page failed.", NULL, NULL ); - } pIndex->fChanged = HB_TRUE; - ulPos = pLst->ulAddr; + ulPage = pLst->nextPage; pLst->fStat = HB_FALSE; pLst = pLst->pNext; #ifdef HB_CDX_DBGUPDT @@ -1145,16 +1156,19 @@ static void hb_cdxIndexDropAvailPage( LPCDXINDEX pIndex ) /* * write index page */ -static void hb_cdxIndexPageWrite( LPCDXINDEX pIndex, HB_ULONG ulPos, HB_BYTE * pBuffer, - HB_USHORT uiSize ) +static void hb_cdxIndexPageWrite( LPCDXINDEX pIndex, HB_ULONG ulPage, + const HB_BYTE * pBuffer, HB_BOOL fHeader ) { + HB_SIZE nSize = fHeader ? CDX_HEADERLEN : CDX_PAGELEN; + if( pIndex->fReadonly ) hb_errInternal( 9101, "hb_cdxIndexPageWrite on readonly database.", NULL, NULL ); if( pIndex->fShared && ! pIndex->lockWrite ) hb_errInternal( 9102, "hb_cdxIndexPageWrite on not locked index file.", NULL, NULL ); hb_cdxIndexLockFlush( pIndex ); - if( hb_fileWriteAt( pIndex->pFile, pBuffer, uiSize, ulPos ) != ( HB_SIZE ) uiSize ) + if( hb_fileWriteAt( pIndex->pFile, pBuffer, nSize, + hb_cdxFilePageOffset( pIndex, ulPage ) ) != nSize ) hb_errInternal( EDBF_WRITE, "Write in index page failed.", NULL, NULL ); pIndex->fChanged = HB_TRUE; #ifdef HB_CDX_DBGUPDT @@ -1165,13 +1179,16 @@ static void hb_cdxIndexPageWrite( LPCDXINDEX pIndex, HB_ULONG ulPos, HB_BYTE * p /* * read index page */ -static void hb_cdxIndexPageRead( LPCDXINDEX pIndex, HB_ULONG ulPos, HB_BYTE * pBuffer, - HB_USHORT uiSize ) +static void hb_cdxIndexPageRead( LPCDXINDEX pIndex, HB_ULONG ulPage, + HB_BYTE * pBuffer, HB_BOOL fHeader ) { + HB_SIZE nSize = fHeader ? CDX_HEADERLEN : CDX_PAGELEN; + if( pIndex->fShared && ! ( pIndex->lockRead || pIndex->lockWrite ) ) hb_errInternal( 9103, "hb_cdxIndexPageRead on not locked index file.", NULL, NULL ); - if( hb_fileReadAt( pIndex->pFile, pBuffer, uiSize, ulPos ) != ( HB_SIZE ) uiSize ) + if( hb_fileReadAt( pIndex->pFile, pBuffer, nSize, + hb_cdxFilePageOffset( pIndex, ulPage ) ) != nSize ) hb_errInternal( EDBF_READ, "hb_cdxIndexPageRead: Read index page failed.", NULL, NULL ); #ifdef HB_CDX_DBGUPDT cdxReadNO++; @@ -1575,7 +1592,7 @@ static HB_ULONG hb_cdxPageGetKeyPage( LPCDXPAGE pPage, int iKey ) /* * get number of duplicated keys from key in leaf index page */ -static HB_BYTE hb_cdxPageGetKeyTrl( LPCDXPAGE pPage, HB_SHORT iKey ) +static HB_BYTE hb_cdxPageGetKeyTrl( LPCDXPAGE pPage, int iKey ) { #ifdef HB_CDX_DBGCODE_EXT if( iKey < 0 || iKey >= pPage->iKeys ) @@ -2267,7 +2284,8 @@ static void hb_cdxPageLoad( LPCDXPAGE pPage ) pPage->pKeyBuf = NULL; pPage->fBufChanged = HB_FALSE; } - hb_cdxIndexPageRead( pPage->TagParent->pIndex, pPage->Page, ( HB_BYTE * ) &pPage->node, sizeof( CDXNODE ) ); + hb_cdxIndexPageRead( pPage->TagParent->pIndex, pPage->Page, + ( HB_BYTE * ) &pPage->node, HB_FALSE ); pPage->PageType = ( HB_BYTE ) HB_GET_LE_UINT16( pPage->node.intNode.attr ); pPage->Left = HB_GET_LE_UINT32( pPage->node.intNode.leftPtr ); pPage->Right = HB_GET_LE_UINT32( pPage->node.intNode.rightPtr ); @@ -2346,7 +2364,8 @@ static void hb_cdxPageStore( LPCDXPAGE pPage ) } #endif } - hb_cdxIndexPageWrite( pPage->TagParent->pIndex, pPage->Page, ( HB_BYTE * ) &pPage->node, sizeof( CDXNODE ) ); + hb_cdxIndexPageWrite( pPage->TagParent->pIndex, pPage->Page, + ( const HB_BYTE * ) &pPage->node, HB_FALSE ); #ifdef HB_CDX_DBGCODE_EXT hb_cdxPageCheckKeys( pPage ); #endif @@ -3414,8 +3433,16 @@ static void hb_cdxTagHeaderStore( LPCDXTAG pTag ) memset( &tagHeader, 0, sizeof( tagHeader ) ); HB_PUT_LE_UINT32( tagHeader.rootPtr, pTag->RootBlock ); HB_PUT_LE_UINT16( tagHeader.keySize, pTag->uiLen ); + HB_PUT_LE_UINT16( tagHeader.headerLen, CDX_HEADERLEN ); + HB_PUT_LE_UINT16( tagHeader.pageLen, CDX_PAGELEN ); tagHeader.indexOpt = pTag->OptFlags; - tagHeader.indexSig = 1; + if( pTag->TagBlock == 0 ) + { + HB_PUT_LE_UINT32( tagHeader.signature, CDX_HARBOUR_SIGNATURE ); + tagHeader.indexSig = pTag->pIndex->fLargeFile ? 0x21 : 0x01; + } + else + tagHeader.indexSig = 0x01; if( ! pTag->AscendKey ) HB_PUT_LE_UINT16( tagHeader.ascendFlg, 1 ); if( pTag->IgnoreCase ) @@ -3443,7 +3470,8 @@ static void hb_cdxTagHeaderStore( LPCDXTAG pTag ) memcpy( tagHeader.keyExpPool + uiKeyLen + 1, pTag->ForExpr, uiForLen ); } } - hb_cdxIndexPageWrite( pTag->pIndex, pTag->TagBlock, ( HB_BYTE * ) &tagHeader, sizeof( CDXTAGHEADER ) ); + hb_cdxIndexPageWrite( pTag->pIndex, pTag->TagBlock, + ( const HB_BYTE * ) &tagHeader, HB_TRUE ); } #if defined( HB_SIXCDX ) @@ -3467,13 +3495,18 @@ static void hb_cdxTagLoad( LPCDXTAG pTag ) HB_ULONG ulRecNo; /* read the page from a file */ - hb_cdxIndexPageRead( pTag->pIndex, pTag->TagBlock, ( HB_BYTE * ) &tagHeader, sizeof( CDXTAGHEADER ) ); + hb_cdxIndexPageRead( pTag->pIndex, pTag->TagBlock, + ( HB_BYTE * ) &tagHeader, HB_TRUE ); uiForPos = HB_GET_LE_UINT16( tagHeader.forExpPos ); uiForLen = HB_GET_LE_UINT16( tagHeader.forExpLen ); uiKeyPos = HB_GET_LE_UINT16( tagHeader.keyExpPos ); uiKeyLen = HB_GET_LE_UINT16( tagHeader.keyExpLen ); + if( pTag->TagBlock == 0 && + HB_GET_LE_UINT32( tagHeader.signature ) == CDX_HARBOUR_SIGNATURE ) + pTag->pIndex->fLargeFile = tagHeader.indexSig == 0x21; + pTag->RootBlock = HB_GET_LE_UINT32( tagHeader.rootPtr ); /* Return if: @@ -3481,8 +3514,8 @@ static void hb_cdxTagLoad( LPCDXTAG pTag ) * invalid root page offset (position inside an index file) * invalid key value length */ - if( pTag->RootBlock == 0 || pTag->RootBlock % CDX_PAGELEN != 0 || - ( HB_FOFFSET ) pTag->RootBlock >= hb_fileSize( pTag->pIndex->pFile ) || + if( pTag->RootBlock == 0 || ! hb_cdxFilePageRootValid( pTag->pIndex, pTag->RootBlock ) || + hb_cdxFilePageOffset( pTag->pIndex, pTag->RootBlock ) >= hb_fileSize( pTag->pIndex->pFile ) || HB_GET_LE_UINT16( tagHeader.keySize ) > CDX_MAXKEY || uiKeyLen + uiForLen > CDX_HEADEREXPLEN || uiForPos + uiForLen > CDX_HEADEREXPLEN || @@ -3534,7 +3567,8 @@ static void hb_cdxTagLoad( LPCDXTAG pTag ) pTag->AscendKey = pTag->UsrAscend = ( HB_GET_LE_UINT16( tagHeader.ascendFlg ) == 0 ); pTag->UsrUnique = HB_FALSE; - if( tagHeader.indexSig == 0x01 ) + + if( tagHeader.indexSig == 0x01 || tagHeader.indexSig == 0x21 ) pTag->IgnoreCase = tagHeader.ignoreCase == 1; else pTag->IgnoreCase = HB_FALSE; @@ -3704,7 +3738,8 @@ static void hb_cdxTagOpen( LPCDXTAG pTag ) if( ! pTag->RootPage ) { - hb_cdxIndexPageRead( pTag->pIndex, pTag->TagBlock, ( HB_BYTE * ) &tagHeader, sizeof( CDXTAGHEADER ) ); + hb_cdxIndexPageRead( pTag->pIndex, pTag->TagBlock, + ( HB_BYTE * ) &tagHeader, HB_TRUE ); pTag->RootBlock = HB_GET_LE_UINT32( tagHeader.rootPtr ); if( pTag->RootBlock && pTag->RootBlock != CDX_DUMMYNODE ) pTag->RootPage = hb_cdxPageNew( pTag, NULL, pTag->RootBlock ); @@ -7760,6 +7795,7 @@ static HB_ERRCODE hb_cdxOrderCreate( CDXAREAP pArea, LPDBORDERCREATEINFO pOrderI if( pIndex->pCompound != NULL ) hb_cdxTagFree( pIndex->pCompound ); pIndex->nextAvail = pIndex->freePage = 0; + pIndex->fLargeFile = ( pIndex->pArea->dbfarea.bLockType == DB_DBFLOCK_HB64 ); hb_cdxIndexCreateStruct( pIndex, szCpndTagName ); } @@ -8596,7 +8632,7 @@ static HB_ERRCODE hb_cdxOrderInfo( CDXAREAP pArea, HB_USHORT uiIndex, LPDBORDERI hb_cdxIndexUnLockRead( pTag->pIndex ); } pInfo->itmResult = hb_itemPutL( pInfo->itmResult, - pTag->pIndex->lockRead > 0 ); + pTag->pIndex->lockRead > 0 ); } else pInfo->itmResult = hb_itemPutL( pInfo->itmResult, HB_FALSE ); @@ -8613,7 +8649,7 @@ static HB_ERRCODE hb_cdxOrderInfo( CDXAREAP pArea, HB_USHORT uiIndex, LPDBORDERI hb_cdxIndexUnLockWrite( pTag->pIndex ); } pInfo->itmResult = hb_itemPutL( pInfo->itmResult, - pTag->pIndex->lockWrite > 0 ); + pTag->pIndex->lockWrite > 0 ); } else pInfo->itmResult = hb_itemPutL( pInfo->itmResult, HB_FALSE ); @@ -8626,7 +8662,7 @@ static HB_ERRCODE hb_cdxOrderInfo( CDXAREAP pArea, HB_USHORT uiIndex, LPDBORDERI if( hb_cdxIndexLockRead( pTag->pIndex ) ) hb_cdxIndexUnLockRead( pTag->pIndex ); pInfo->itmResult = hb_itemPutNInt( pInfo->itmResult, - pTag->pIndex->ulVersion ); + pTag->pIndex->ulVersion ); } else pInfo->itmResult = hb_itemPutNI( pInfo->itmResult, 0 ); @@ -8634,12 +8670,12 @@ static HB_ERRCODE hb_cdxOrderInfo( CDXAREAP pArea, HB_USHORT uiIndex, LPDBORDERI case DBOI_SHARED: pInfo->itmResult = hb_itemPutL( pInfo->itmResult, - pTag && pTag->pIndex->fShared ); + pTag && pTag->pIndex->fShared ); break; case DBOI_ISREADONLY: pInfo->itmResult = hb_itemPutL( pInfo->itmResult, - pTag && pTag->pIndex->fReadonly ); + pTag && pTag->pIndex->fReadonly ); break; case DBOI_ISMULTITAG: @@ -8648,12 +8684,13 @@ static HB_ERRCODE hb_cdxOrderInfo( CDXAREAP pArea, HB_USHORT uiIndex, LPDBORDERI break; case DBOI_LARGEFILE: - pInfo->itmResult = hb_itemPutL( pInfo->itmResult, HB_FALSE ); + pInfo->itmResult = hb_itemPutL( pInfo->itmResult, + pTag && pTag->pIndex->fLargeFile ); break; case DBOI_INDEXTYPE: pInfo->itmResult = hb_itemPutNI( pInfo->itmResult, pTag ? - DBOI_TYPE_COMPOUND : DBOI_TYPE_UNDEF ); + DBOI_TYPE_COMPOUND : DBOI_TYPE_UNDEF ); break; default: @@ -9489,7 +9526,7 @@ static void hb_cdxTagDoIndex( LPCDXTAG pTag, HB_BOOL fReindex ) HB_ULONG ulStartRec = 0, ulNextCount = 0; HB_BOOL fDirectRead, fUseFilter = HB_FALSE; HB_BYTE * pSaveRecBuff = pArea->dbfarea.pRecord, cTemp[8]; - int iRecBuff = 0, iRecBufSize = USHRT_MAX / pArea->dbfarea.uiRecordLen, iRec; + int iRecBuff = 0, iRecBufSize, iRec; pForItem = pTag->pForItem; if( pTag->nField ) @@ -9537,7 +9574,9 @@ static void hb_cdxTagDoIndex( LPCDXTAG pTag, HB_BOOL fReindex ) } } } - fDirectRead = ! hb_setGetStrictRead() && /* ! pArea->dbfarea.area.lpdbRelations && */ + + iRecBufSize = ( USHRT_MAX + 1 ) / pArea->dbfarea.uiRecordLen; + fDirectRead = ! hb_setGetStrictRead() && iRecBufSize > 1 && /* ! pArea->dbfarea.area.lpdbRelations && */ ( ! pArea->dbfarea.area.lpdbOrdCondInfo || pArea->dbfarea.area.lpdbOrdCondInfo->fAll || ( pArea->uiTag == 0 && ! fUseFilter ) ); @@ -9568,16 +9607,24 @@ static void hb_cdxTagDoIndex( LPCDXTAG pTag, HB_BOOL fReindex ) break; if( iRecBuff == 0 || iRecBuff >= iRecBufSize ) { + HB_SIZE nSize; + if( ulRecCount - ulRecNo >= ( HB_ULONG ) iRecBufSize ) iRec = iRecBufSize; else iRec = ulRecCount - ulRecNo + 1; if( ulNextCount > 0 && ulNextCount < ( HB_ULONG ) iRec ) iRec = ( int ) ulNextCount; - hb_fileReadAt( pArea->dbfarea.pDataFile, pSort->pRecBuff, pArea->dbfarea.uiRecordLen * iRec, - ( HB_FOFFSET ) pArea->dbfarea.uiHeaderLen + - ( HB_FOFFSET ) ( ulRecNo - 1 ) * - ( HB_FOFFSET ) pArea->dbfarea.uiRecordLen ); + nSize = iRec * pArea->dbfarea.uiRecordLen; + if( hb_fileReadAt( pArea->dbfarea.pDataFile, pSort->pRecBuff, nSize, + ( HB_FOFFSET ) pArea->dbfarea.uiHeaderLen + + ( HB_FOFFSET ) ( ulRecNo - 1 ) * + ( HB_FOFFSET ) pArea->dbfarea.uiRecordLen ) != nSize ) + { + hb_cdxErrorRT( pTag->pIndex->pArea, EG_READ, EDBF_READ, + pTag->pIndex->szFileName, hb_fsError(), 0, NULL ); + break; + } iRecBuff = 0; } pArea->dbfarea.pRecord = pSort->pRecBuff + iRecBuff * pArea->dbfarea.uiRecordLen; diff --git a/src/rdd/dbfnsx/dbfnsx1.c b/src/rdd/dbfnsx/dbfnsx1.c index 45e92c8271..3a93257007 100644 --- a/src/rdd/dbfnsx/dbfnsx1.c +++ b/src/rdd/dbfnsx/dbfnsx1.c @@ -76,7 +76,7 @@ static HB_USHORT s_uiRddId; #define HB_INTCAST int #define hb_nsxKeyFree( K ) hb_xfree( K ) -#define hb_nsxFileOffset( I, B ) ( ( B ) << ( ( I )->LargeFile ? NSX_PAGELEN_BITS : 0 ) ) +#define hb_nsxFileOffset( I, B ) ( ( HB_FOFFSET ) ( B ) << ( ( I )->LargeFile ? NSX_PAGELEN_BITS : 0 ) ) #define hb_nsxGetRecSize( r ) ( ( r ) < 0x10000 ? 2 : ( ( r ) < 0x1000000 ? 3 : 4 ) ) #define hb_nsxPageBuffer( p ) ( ( p )->data.buffer ) #define hb_nsxIsLeaf( p ) ( ( ( p )->data.buffer[ 0 ] & NSX_LEAFPAGE ) != 0 ) @@ -5897,13 +5897,12 @@ static HB_ERRCODE hb_nsxTagCreate( LPTAGINFO pTag, HB_BOOL fReindex ) } } - fDirectRead = ! hb_setGetStrictRead() && /* ! pArea->dbfarea.area.lpdbRelations && */ - ( ! pArea->dbfarea.area.lpdbOrdCondInfo || pArea->dbfarea.area.lpdbOrdCondInfo->fAll || - ( pArea->lpCurTag == NULL && ! fUseFilter ) ); - pSort->ulSizeIO = ( 1 << 16 ) / NSX_PAGELEN; pSort->pBuffIO = ( HB_UCHAR * ) hb_xgrab( pSort->ulSizeIO * NSX_PAGELEN ); iRecBufSize = ( pSort->ulSizeIO * NSX_PAGELEN ) / pArea->dbfarea.uiRecordLen; + fDirectRead = ! hb_setGetStrictRead() && iRecBufSize > 1 && /* ! pArea->dbfarea.area.lpdbRelations && */ + ( ! pArea->dbfarea.area.lpdbOrdCondInfo || pArea->dbfarea.area.lpdbOrdCondInfo->fAll || + ( pArea->lpCurTag == NULL && ! fUseFilter ) ); if( ulStartRec == 0 && pArea->lpCurTag == NULL ) ulStartRec = 1; @@ -5935,16 +5934,24 @@ static HB_ERRCODE hb_nsxTagCreate( LPTAGINFO pTag, HB_BOOL fReindex ) break; if( iRecBuff == 0 || iRecBuff >= iRecBufSize ) { + HB_SIZE nSize; + if( ulRecCount - ulRecNo >= ( HB_ULONG ) iRecBufSize ) iRec = iRecBufSize; else iRec = ulRecCount - ulRecNo + 1; if( ulNextCount > 0 && ulNextCount < ( HB_ULONG ) iRec ) iRec = ( int ) ulNextCount; - hb_fileReadAt( pArea->dbfarea.pDataFile, pSort->pBuffIO, pArea->dbfarea.uiRecordLen * iRec, - ( HB_FOFFSET ) pArea->dbfarea.uiHeaderLen + - ( HB_FOFFSET ) ( ulRecNo - 1 ) * - ( HB_FOFFSET ) pArea->dbfarea.uiRecordLen ); + nSize = iRec * pArea->dbfarea.uiRecordLen; + if( hb_fileReadAt( pArea->dbfarea.pDataFile, pSort->pBuffIO, nSize, + ( HB_FOFFSET ) pArea->dbfarea.uiHeaderLen + + ( HB_FOFFSET ) ( ulRecNo - 1 ) * + ( HB_FOFFSET ) pArea->dbfarea.uiRecordLen ) != nSize ) + { + hb_nsxErrorRT( pTag->pIndex->pArea, EG_READ, EDBF_READ, + pTag->pIndex->IndexName, hb_fsError(), 0, NULL ); + break; + } iRecBuff = 0; } pArea->dbfarea.pRecord = pSort->pBuffIO + iRecBuff * pArea->dbfarea.uiRecordLen; diff --git a/src/rdd/dbfntx/dbfntx1.c b/src/rdd/dbfntx/dbfntx1.c index fa1e0a6635..9f13b4e7e2 100644 --- a/src/rdd/dbfntx/dbfntx1.c +++ b/src/rdd/dbfntx/dbfntx1.c @@ -157,7 +157,7 @@ static HB_USHORT s_uiRddId; #define HB_INTCAST int #define hb_ntxKeyFree( K ) hb_xfree( K ) -#define hb_ntxFileOffset( I, B ) ( ( B ) << ( ( I )->LargeFile ? NTXBLOCKBITS : 0 ) ) +#define hb_ntxFileOffset( I, B ) ( ( HB_FOFFSET ) ( B ) << ( ( I )->LargeFile ? NTXBLOCKBITS : 0 ) ) #define hb_ntxPageBuffer( p ) ( ( p )->buffer ) /* @@ -5402,13 +5402,12 @@ static HB_ERRCODE hb_ntxTagCreate( LPTAGINFO pTag, HB_BOOL fReindex ) } } - fDirectRead = ! hb_setGetStrictRead() && /* ! pArea->dbfarea.area.lpdbRelations && */ - ( ! pArea->dbfarea.area.lpdbOrdCondInfo || pArea->dbfarea.area.lpdbOrdCondInfo->fAll || - ( pArea->lpCurTag == NULL && ! fUseFilter ) ); - pSort->ulSizeIO = ( 1 << 16 ) / NTXBLOCKSIZE; pSort->pBuffIO = ( HB_BYTE * ) hb_xgrab( pSort->ulSizeIO * NTXBLOCKSIZE ); iRecBufSize = ( pSort->ulSizeIO * NTXBLOCKSIZE ) / pArea->dbfarea.uiRecordLen; + fDirectRead = ! hb_setGetStrictRead() && iRecBufSize > 1 && /* ! pArea->dbfarea.area.lpdbRelations && */ + ( ! pArea->dbfarea.area.lpdbOrdCondInfo || pArea->dbfarea.area.lpdbOrdCondInfo->fAll || + ( pArea->lpCurTag == NULL && ! fUseFilter ) ); if( ulStartRec == 0 && pArea->lpCurTag == NULL ) ulStartRec = 1; @@ -5440,16 +5439,24 @@ static HB_ERRCODE hb_ntxTagCreate( LPTAGINFO pTag, HB_BOOL fReindex ) break; if( iRecBuff == 0 || iRecBuff >= iRecBufSize ) { + HB_SIZE nSize; + if( ulRecCount - ulRecNo >= ( HB_ULONG ) iRecBufSize ) iRec = iRecBufSize; else iRec = ulRecCount - ulRecNo + 1; if( ulNextCount > 0 && ulNextCount < ( HB_ULONG ) iRec ) iRec = ( int ) ulNextCount; - hb_fileReadAt( pArea->dbfarea.pDataFile, pSort->pBuffIO, pArea->dbfarea.uiRecordLen * iRec, - ( HB_FOFFSET ) pArea->dbfarea.uiHeaderLen + - ( HB_FOFFSET ) ( ulRecNo - 1 ) * - ( HB_FOFFSET ) pArea->dbfarea.uiRecordLen ); + nSize = iRec * pArea->dbfarea.uiRecordLen; + if( hb_fileReadAt( pArea->dbfarea.pDataFile, pSort->pBuffIO, nSize, + ( HB_FOFFSET ) pArea->dbfarea.uiHeaderLen + + ( HB_FOFFSET ) ( ulRecNo - 1 ) * + ( HB_FOFFSET ) pArea->dbfarea.uiRecordLen ) != nSize ) + { + hb_ntxErrorRT( pTag->Owner->Owner, EG_READ, EDBF_READ, + pTag->Owner->IndexName, hb_fsError(), 0, NULL ); + break; + } iRecBuff = 0; } pArea->dbfarea.pRecord = pSort->pBuffIO + iRecBuff * pArea->dbfarea.uiRecordLen;