diff --git a/ChangeLog.txt b/ChangeLog.txt index c57f7593f1..a04dca9175 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -10,6 +10,36 @@ * Change, ! Fix, % Optimization, + Addition, - Removal, ; Comment */ +2016-01-04 21:25 UTC+0100 Przemyslaw Czerpak (druzus/at/poczta.onet.pl) + * src/rtl/hbznet.c + * src/rtl/hbzsock.c + ! cleanup + + * include/hbrdddel.h + * include/hbrddsdf.h + * src/rdd/delim1.c + * src/rdd/sdf1.c + + added support for any standard EOL combination to DELIM and SDF RDDs, + Now LF, CRLF, CR and LFCR EOLs are supported and automatically + recognized during import if one of such EOLs is also set in HVM by + _SET_EOL. + % use 8192 bytes read ahead buffer to speed up import from DELIM and + SDF RDDs + + modified file IO operations to work with stream only (no support for + file seek) Harbour File IO drivers + % optimized some File IO operations + * strip all EOF (^Z) characters from imported data. It's not strictly + Cl*pper compatible because Cl*pper interrupts import on first ^Z + character anyhow it was always the source of problems when .txt + files are concatenated by tools like 'cat' in *nixes so I decided + to change it. + + added support RECORD clause command in commands like: + APPEND FROM "data.txt" RECORD (nRec) DELIMITED + APPEND FROM "data.txt" RECORD (nRec) SDF + It's Harbour extension - Cl*pper does not support it. + + generate RTE when some unsupported in transfer RDDs methods are + called + 2016-01-03 09:24 UTC-0800 Pritpal Bedi (bedipritpal/at/hotmail.com) * contrib/gtwvg/wvgcore.c * contrib/gtwvg/wvgcuig.c diff --git a/include/hbrdddel.h b/include/hbrdddel.h index 2480496c04..94097f3cdb 100644 --- a/include/hbrdddel.h +++ b/include/hbrdddel.h @@ -76,31 +76,29 @@ typedef struct _DELIMAREA * example. */ - PHB_FILE pFile; /* Data file handle */ - char * szFileName; /* Name of data file */ - char * szEol; /* EOL marker */ - HB_USHORT uiEolLen; /* Size of EOL marker */ - char cDelim; /* Character field delimiter */ - char cSeparator; /* Field separator */ - HB_USHORT uiRecordLen; /* Size of record */ - HB_USHORT * pFieldOffset; /* Pointer to field offset array */ - HB_BYTE * pRecord; /* Buffer of record data */ - HB_BYTE * pBuffer; /* Read/Write */ - HB_SIZE nBufferSize; /* IO buffer size */ - HB_SIZE nBufferRead; /* Number of bytes in read buffer */ - HB_SIZE nBufferIndex; /* Index to read read buffer */ - HB_FOFFSET nRecordOffset; /* Current record offest */ - HB_FOFFSET nNextOffset; /* Next record offest */ - HB_FOFFSET nFileSize; /* File table size in export mode */ - HB_FOFFSET nBufferStart; /* Start offset of read buffer */ - HB_ULONG ulRecNo; /* Current record */ - HB_ULONG ulRecCount; /* Number of records (in export) */ - HB_BOOL fTransRec; /* Can put whole records */ - HB_BOOL fFlush; /* Data was written to table and not commited */ - HB_BOOL fShared; /* Shared file */ - HB_BOOL fReadonly; /* Read only file */ - HB_BOOL fPositioned; /* Positioned record */ - HB_BOOL fRecordChanged; /* Record changed */ + PHB_FILE pFile; /* Data file handle */ + char * szFileName; /* Name of data file */ + char * szEol; /* EOL marker */ + HB_USHORT uiEolLen; /* Size of EOL marker */ + char cDelim; /* Character field delimiter */ + char cSeparator; /* Field separator */ + HB_USHORT uiRecordLen; /* Size of record */ + HB_USHORT * pFieldOffset; /* Pointer to field offset array */ + HB_BYTE * pRecord; /* Buffer of record data */ + HB_BYTE * pBuffer; /* Read/Write */ + HB_SIZE nBufferSize; /* IO buffer size */ + HB_SIZE nBufferRead; /* Number of bytes in read buffer */ + HB_SIZE nBufferAtRead; /* The index in the buffer where we should read next peace of data */ + HB_SIZE nBufferIndex; /* Index to read read buffer */ + HB_ULONG ulRecNo; /* Current record */ + HB_ULONG ulRecCount; /* Number of records (in export) */ + HB_BOOL fTransRec; /* Can put whole records */ + HB_BOOL fFlush; /* Data was written to table and not commited */ + HB_BOOL fShared; /* Shared file */ + HB_BOOL fReadonly; /* Read only file */ + HB_BOOL fPositioned; /* Positioned record */ + HB_BOOL fRecordChanged; /* Record changed */ + HB_BOOL fAnyEol; /* Check for CRLF, LF, CR and LFCR EOLs */ } DELIMAREA; typedef DELIMAREA * LPDELIMAREA; diff --git a/include/hbrddsdf.h b/include/hbrddsdf.h index 884d5f8019..c6608504a6 100644 --- a/include/hbrddsdf.h +++ b/include/hbrddsdf.h @@ -76,24 +76,26 @@ typedef struct _SDFAREA * example. */ - PHB_FILE pFile; /* Data file handle */ - char * szFileName; /* Name of data file */ - char * szEol; /* EOL marker */ - HB_USHORT uiEolLen; /* Size of EOL marker */ - HB_USHORT uiRecordLen; /* Size of record */ - HB_USHORT * pFieldOffset; /* Pointer to field offset array */ - HB_BYTE * pRecord; /* Buffer of record data */ - HB_FOFFSET nRecordOffset; /* Current record offest */ - HB_FOFFSET nNextOffset; /* Next record offest */ - HB_FOFFSET nFileSize; /* File table size in export mode */ - HB_ULONG ulRecNo; /* Current record */ - HB_ULONG ulRecCount; /* Number of records (in export) */ - HB_BOOL fTransRec; /* Can put whole records */ - HB_BOOL fFlush; /* Data was written to SDF and not commited */ - HB_BOOL fShared; /* Shared file */ - HB_BOOL fReadonly; /* Read only file */ - HB_BOOL fPositioned; /* Positioned record */ - HB_BOOL fRecordChanged; /* Record changed */ + PHB_FILE pFile; /* Data file handle */ + char * szFileName; /* Name of data file */ + char * szEol; /* EOL marker */ + HB_USHORT uiEolLen; /* Size of EOL marker */ + HB_USHORT uiRecordLen; /* Size of record */ + HB_USHORT * pFieldOffset; /* Pointer to field offset array */ + HB_BYTE * pRecord; /* Buffer of record data */ + HB_BYTE * pBuffer; /* Read/Write */ + HB_SIZE nBufferSize; /* IO buffer size */ + HB_SIZE nBufferRead; /* Number of bytes in read buffer */ + HB_SIZE nBufferIndex; /* Index to read read buffer */ + HB_ULONG ulRecNo; /* Current record */ + HB_ULONG ulRecCount; /* Number of records (in export) */ + HB_BOOL fTransRec; /* Can put whole records */ + HB_BOOL fFlush; /* Data was written to SDF and not commited */ + HB_BOOL fShared; /* Shared file */ + HB_BOOL fReadonly; /* Read only file */ + HB_BOOL fPositioned; /* Positioned record */ + HB_BOOL fRecordChanged; /* Record changed */ + HB_BOOL fAnyEol; /* Check for CRLF, LF, CR and LFCR EOLs */ } SDFAREA; typedef SDFAREA * LPSDFAREA; diff --git a/src/rdd/delim1.c b/src/rdd/delim1.c index 30eaa0c774..39bd1da852 100644 --- a/src/rdd/delim1.c +++ b/src/rdd/delim1.c @@ -76,20 +76,26 @@ static void hb_delimInitArea( DELIMAREAP pArea, char * szFileName ) if( ! szEol || ! szEol[ 0 ] ) szEol = hb_conNewLine(); pArea->szEol = hb_strdup( szEol ); - pArea->uiEolLen = ( HB_USHORT ) strlen( pArea->szEol ); + pArea->uiEolLen = ( HB_USHORT ) strlen( szEol ); + pArea->fAnyEol = ( szEol[ 0 ] == '\n' || szEol[ 0 ] == '\r' ) && + ( pArea->uiEolLen == 1 || + ( pArea->uiEolLen == 2 && szEol[ 0 ] != szEol[ 1 ] && + ( szEol[ 1 ] == '\n' || szEol[ 1 ] == '\r' ) ) ); /* allocate record buffer, one additional byte is for deleted flag */ pArea->pRecord = ( HB_BYTE * ) hb_xgrab( pArea->uiRecordLen + 1 ); /* pseudo deleted flag */ *pArea->pRecord++ = ' '; - /* Allocate IO buffer */ - pArea->nBufferSize += pArea->uiEolLen; + /* allocate IO buffer */ + pArea->nBufferSize += pArea->fAnyEol ? 2 : pArea->uiEolLen; + if( pArea->fReadonly && pArea->nBufferSize < 8192 ) + pArea->nBufferSize = 8192; pArea->pBuffer = ( HB_BYTE * ) hb_xgrab( pArea->nBufferSize ); pArea->ulRecCount = 0; - pArea->nFileSize = 0; - pArea->nBufferRead = pArea->nBufferIndex = 0; + pArea->nBufferIndex = pArea->nBufferRead = pArea->nBufferSize; + pArea->nBufferAtRead = pArea->nBufferSize - HB_MAX( pArea->uiEolLen, 2 ); } static void hb_delimClearRecordBuffer( DELIMAREAP pArea ) @@ -192,44 +198,67 @@ static HB_SIZE hb_delimEncodeBuffer( DELIMAREAP pArea ) static int hb_delimNextChar( DELIMAREAP pArea ) { - if( pArea->nBufferIndex + pArea->uiEolLen >= pArea->nBufferRead && - ( pArea->nBufferRead == 0 || - pArea->nBufferRead >= pArea->nBufferSize - 1 ) ) + for( ;; ) { - HB_SIZE nLeft = pArea->nBufferRead - pArea->nBufferIndex; + char ch; - if( nLeft ) - memcpy( pArea->pBuffer, - pArea->pBuffer + pArea->nBufferIndex, nLeft ); - pArea->nBufferStart += pArea->nBufferIndex; - pArea->nBufferIndex = 0; - pArea->nBufferRead = hb_fileReadAt( pArea->pFile, - pArea->pBuffer + nLeft, - pArea->nBufferSize - nLeft, - pArea->nBufferStart + nLeft ); - if( pArea->nBufferRead == ( HB_SIZE ) FS_ERROR ) - pArea->nBufferRead = 0; - if( pArea->nBufferRead > 0 && - pArea->pBuffer[ pArea->nBufferRead + nLeft - 1 ] == '\032' ) - pArea->nBufferRead--; - pArea->nBufferRead += nLeft; - } + if( pArea->nBufferIndex >= pArea->nBufferAtRead && + pArea->nBufferRead == pArea->nBufferSize ) + { + HB_SIZE nLeft = pArea->nBufferRead - pArea->nBufferIndex; - if( pArea->nBufferIndex + pArea->uiEolLen <= pArea->nBufferRead && - memcmp( pArea->pBuffer + pArea->nBufferIndex, - pArea->szEol, pArea->uiEolLen ) == 0 ) - { - pArea->nBufferIndex += pArea->uiEolLen; - pArea->nNextOffset = pArea->nBufferStart + pArea->nBufferIndex; - return -1; - } - else if( pArea->nBufferIndex < pArea->nBufferRead ) - { - return pArea->pBuffer[ pArea->nBufferIndex++ ]; - } + if( nLeft ) + memmove( pArea->pBuffer, + pArea->pBuffer + pArea->nBufferIndex, nLeft ); + pArea->nBufferIndex = 0; + pArea->nBufferRead = hb_fileRead( pArea->pFile, + pArea->pBuffer + nLeft, + pArea->nBufferSize - nLeft, -1 ); + if( pArea->nBufferRead == ( HB_SIZE ) FS_ERROR ) + pArea->nBufferRead = 0; + pArea->nBufferRead += nLeft; + } - pArea->nNextOffset = ( HB_FOFFSET ) -1; - return -2; + if( pArea->nBufferIndex >= pArea->nBufferRead ) + return -2; + + ch = pArea->pBuffer[ pArea->nBufferIndex++ ]; + + if( pArea->fAnyEol ) + { + if( ch == '\r' || ch == '\n' ) + { + if( pArea->nBufferIndex < pArea->nBufferRead && + pArea->pBuffer[ pArea->nBufferIndex ] != ch && + ( pArea->pBuffer[ pArea->nBufferIndex ] == '\r' || + pArea->pBuffer[ pArea->nBufferIndex ] == '\n' ) ) + pArea->nBufferIndex++; + return -1; + } + } + else if( ch == pArea->szEol[ 0 ] ) + { + if( pArea->uiEolLen == 1 ) + return -1; + else if( pArea->nBufferRead - pArea->nBufferIndex >= + ( HB_SIZE ) pArea->uiEolLen - 1 && + memcmp( pArea->pBuffer + pArea->nBufferIndex, + pArea->szEol + 1, pArea->uiEolLen - 1 ) == 0 ) + { + pArea->nBufferIndex += pArea->uiEolLen - 1; + return -1; + } + } + if( ch != '\032' ) + return ch; + + /* Cl*pper stops farther file processing when first EOF + character is read [druzus] */ +#ifdef HB_CLP_STRICT + pArea->nBufferRead = pArea->nBufferIndex = 0; + return -2; +#endif + } } /* @@ -237,47 +266,38 @@ static int hb_delimNextChar( DELIMAREAP pArea ) */ static HB_ERRCODE hb_delimReadRecord( DELIMAREAP pArea ) { - HB_USHORT uiField, uiLen, uiSize; - HB_TYPE uiType; - LPFIELD pField; - HB_BYTE * pFieldBuf, buffer[ 256 ]; - char cStop; + + HB_USHORT uiField; int ch = 0; HB_TRACE( HB_TR_DEBUG, ( "hb_delimReadRecord(%p)", pArea ) ); - if( pArea->nBufferStart <= pArea->nRecordOffset && - pArea->nBufferStart + ( HB_FOFFSET ) pArea->nBufferRead > pArea->nRecordOffset ) - { - pArea->nBufferIndex = ( HB_SIZE ) ( pArea->nRecordOffset - pArea->nBufferStart ); - } - else - { - pArea->nBufferStart = pArea->nRecordOffset; - pArea->nBufferRead = pArea->nBufferIndex = 0; - } + pArea->area.fEof = HB_TRUE; /* clear the record buffer */ hb_delimClearRecordBuffer( pArea ); for( uiField = 0; uiField < pArea->area.uiFieldCount; ++uiField ) { - pField = pArea->area.lpFields + uiField; - uiType = pField->uiType; + LPFIELD pField = pArea->area.lpFields + uiField; + HB_USHORT uiType = pField->uiType; + if( uiType == HB_FT_STRING || uiType == HB_FT_LOGICAL || uiType == HB_FT_DATE || uiType == HB_FT_TIMESTAMP || uiType == HB_FT_LONG ) { - uiSize = 0; - uiLen = pField->uiLen; - pFieldBuf = pArea->pRecord + pArea->pFieldOffset[ uiField ]; + HB_USHORT uiLen = pField->uiLen, uiSize = 0; + HB_BYTE * pFieldBuf = pArea->pRecord + pArea->pFieldOffset[ uiField ], + buffer[ 256 ]; + char cStop; + + ch = hb_delimNextChar( pArea ); + if( ch != -2 ) + pArea->area.fEof = HB_FALSE; /* ignore leading spaces */ - do - { + while( ch == ' ' ) ch = hb_delimNextChar( pArea ); - } - while( ch == ' ' ); /* set the stop character */ if( pArea->cDelim && ch == pArea->cDelim ) @@ -289,8 +309,8 @@ static HB_ERRCODE hb_delimReadRecord( DELIMAREAP pArea ) cStop = pArea->cSeparator; /* - * Clipper uses differ rules for character fields, they - * can be terminated only with valid stop character, when + * Clipper uses different rules for character fields, they + * can be terminated only with valid stop character when * other fields also by length */ if( pField->uiType == HB_FT_STRING || @@ -363,18 +383,7 @@ static HB_ERRCODE hb_delimReadRecord( DELIMAREAP pArea ) while( ch >= 0 ) ch = hb_delimNextChar( pArea ); - if( ch == -2 && - pArea->nRecordOffset == ( HB_FOFFSET ) ( pArea->nBufferStart + - pArea->nBufferIndex ) ) - { - pArea->area.fEof = HB_TRUE; - pArea->fPositioned = HB_FALSE; - } - else - { - pArea->area.fEof = HB_FALSE; - pArea->fPositioned = HB_TRUE; - } + pArea->fPositioned = ! pArea->area.fEof; return HB_SUCCESS; } @@ -385,18 +394,8 @@ static HB_ERRCODE hb_delimNextRecord( DELIMAREAP pArea ) if( pArea->fPositioned ) { - if( pArea->nNextOffset == ( HB_FOFFSET ) -1 ) - { - pArea->area.fEof = HB_TRUE; - pArea->fPositioned = HB_FALSE; - hb_delimClearRecordBuffer( pArea ); - } - else - { - pArea->ulRecNo++; - pArea->nRecordOffset = pArea->nNextOffset; - return hb_delimReadRecord( pArea ); - } + pArea->ulRecNo++; + return hb_delimReadRecord( pArea ); } return HB_SUCCESS; } @@ -405,6 +404,43 @@ static HB_ERRCODE hb_delimNextRecord( DELIMAREAP pArea ) * -- DELIM METHODS -- */ +/* + * Position cursor at a specific physical record. + */ +static HB_ERRCODE hb_delimGoTo( DELIMAREAP pArea, HB_ULONG ulRecNo ) +{ + HB_TRACE( HB_TR_DEBUG, ( "hb_delimGoTo(%p, %lu)", pArea, ulRecNo ) ); + +#ifndef HB_CLP_STRICT + if( pArea->fReadonly && ulRecNo >= pArea->ulRecNo ) + { + while( pArea->ulRecNo < ulRecNo && pArea->fPositioned ) + { + if( hb_delimNextRecord( pArea ) != HB_SUCCESS ) + return HB_FAILURE; + } + return HB_SUCCESS; + } +#endif + /* generate RTE */ + return SUPER_GOTO( &pArea->area, ulRecNo ); +} + +/* + * Position the cursor to a specific, physical identity. + */ +static HB_ERRCODE hb_delimGoToId( DELIMAREAP pArea, PHB_ITEM pItem ) +{ + HB_TRACE( HB_TR_DEBUG, ( "hb_delimGoToId(%p, %p)", pArea, pItem ) ); + +#ifndef HB_CLP_STRICT + if( HB_IS_NUMERIC( pItem ) ) + return SELF_GOTO( &pArea->area, hb_itemGetNL( pItem ) ); +#endif + /* generate RTE */ + return SUPER_GOTOID( &pArea->area, pItem ); +} + /* * Position cursor at the first record. */ @@ -418,10 +454,16 @@ static HB_ERRCODE hb_delimGoTop( DELIMAREAP pArea ) pArea->area.fTop = HB_TRUE; pArea->area.fBottom = HB_FALSE; - pArea->nRecordOffset = 0; - pArea->ulRecNo = 1; - if( hb_delimReadRecord( pArea ) != HB_SUCCESS ) - return HB_FAILURE; + if( pArea->ulRecNo != 1 ) + { + if( pArea->ulRecNo != 0 || ! pArea->fReadonly ) + /* generate RTE */ + return SUPER_GOTOP( &pArea->area ); + + pArea->ulRecNo = 1; + if( hb_delimReadRecord( pArea ) != HB_SUCCESS ) + return HB_FAILURE; + } return SELF_SKIPFILTER( &pArea->area, 1 ); } @@ -436,8 +478,9 @@ static HB_ERRCODE hb_delimSkipRaw( DELIMAREAP pArea, HB_LONG lToSkip ) if( SELF_GOCOLD( &pArea->area ) != HB_SUCCESS ) return HB_FAILURE; - if( lToSkip != 1 ) - return HB_FAILURE; + if( lToSkip != 1 || ! pArea->fReadonly ) + /* generate RTE */ + return SUPER_SKIPRAW( &pArea->area, lToSkip ); else return hb_delimNextRecord( pArea ); } @@ -524,7 +567,6 @@ static HB_ERRCODE hb_delimAppend( DELIMAREAP pArea, HB_BOOL fUnLockAll ) if( SELF_GOHOT( &pArea->area ) != HB_SUCCESS ) return HB_FAILURE; - pArea->nRecordOffset = pArea->nFileSize; pArea->ulRecNo = ++pArea->ulRecCount; pArea->area.fEof = HB_FALSE; pArea->fPositioned = HB_TRUE; @@ -868,8 +910,7 @@ static HB_ERRCODE hb_delimGoCold( DELIMAREAP pArea ) { HB_SIZE nSize = hb_delimEncodeBuffer( pArea ); - if( hb_fileWriteAt( pArea->pFile, pArea->pBuffer, nSize, - pArea->nRecordOffset ) != nSize ) + if( hb_fileWrite( pArea->pFile, pArea->pBuffer, nSize, -1 ) != nSize ) { PHB_ITEM pError = hb_errNew(); @@ -882,8 +923,6 @@ static HB_ERRCODE hb_delimGoCold( DELIMAREAP pArea ) hb_itemRelease( pError ); return HB_FAILURE; } - pArea->nFileSize += nSize; - pArea->nNextOffset = pArea->nFileSize; pArea->fRecordChanged = HB_FALSE; pArea->fFlush = HB_TRUE; } @@ -924,15 +963,10 @@ static HB_ERRCODE hb_delimFlush( DELIMAREAP pArea ) errCode = SELF_GOCOLD( &pArea->area ); - if( pArea->fFlush ) + if( pArea->fFlush && hb_setGetHardCommit() ) { - if( hb_setGetEOF() ) - hb_fileWriteAt( pArea->pFile, "\032", 1, pArea->nFileSize ); - if( hb_setGetHardCommit() ) - { - hb_fileCommit( pArea->pFile ); - pArea->fFlush = HB_FALSE; - } + hb_fileCommit( pArea->pFile ); + pArea->fFlush = HB_FALSE; } return errCode; @@ -1025,7 +1059,7 @@ static HB_ERRCODE hb_delimInfo( DELIMAREAP pArea, HB_USHORT uiIndex, PHB_ITEM pI break; } case DBI_FULLPATH: - hb_itemPutC( pItem, pArea->szFileName); + hb_itemPutC( pItem, pArea->szFileName ); break; case DBI_FILEHANDLE: @@ -1249,16 +1283,23 @@ static HB_ERRCODE hb_delimClose( DELIMAREAP pArea ) { HB_TRACE( HB_TR_DEBUG, ( "hb_delimClose(%p)", pArea ) ); - SUPER_CLOSE( &pArea->area ); - /* Update record and unlock records */ if( pArea->pFile ) { + SELF_GOCOLD( &pArea->area ); + + if( ! pArea->fReadonly && hb_setGetEOF() ) + { + hb_fileWrite( pArea->pFile, "\032", 1, -1 ); + pArea->fFlush = HB_TRUE; + } SELF_FLUSH( &pArea->area ); hb_fileClose( pArea->pFile ); pArea->pFile = NULL; } + SUPER_CLOSE( &pArea->area ); + if( pArea->pFieldOffset ) { hb_xfree( pArea->pFieldOffset ); @@ -1294,9 +1335,9 @@ static HB_ERRCODE hb_delimClose( DELIMAREAP pArea ) static HB_ERRCODE hb_delimCreate( DELIMAREAP pArea, LPDBOPENINFO pCreateInfo ) { PHB_ITEM pError = NULL; - PHB_FNAME pFileName; HB_ERRCODE errCode; HB_BOOL fRetry; + PHB_FNAME pFileName; char szFileName[ HB_PATH_MAX ]; HB_TRACE( HB_TR_DEBUG, ( "hb_delimCreate(%p,%p)", pArea, pCreateInfo ) ); @@ -1333,8 +1374,7 @@ static HB_ERRCODE hb_delimCreate( DELIMAREAP pArea, LPDBOPENINFO pCreateInfo ) { pArea->pFile = hb_fileExtOpen( szFileName, NULL, FO_READWRITE | FO_EXCLUSIVE | FXO_TRUNCATE | - FXO_DEFAULTS | FXO_SHARELOCK | FXO_COPYNAME | - FXO_NOSEEKPOS, + FXO_DEFAULTS | FXO_SHARELOCK | FXO_COPYNAME, NULL, pError ); if( ! pArea->pFile ) { @@ -1369,9 +1409,12 @@ static HB_ERRCODE hb_delimCreate( DELIMAREAP pArea, LPDBOPENINFO pCreateInfo ) } hb_delimInitArea( pArea, szFileName ); + pArea->ulRecNo = 1; + pArea->area.fEof = HB_TRUE; + pArea->fPositioned = HB_FALSE; + hb_delimClearRecordBuffer( pArea ); - /* Position cursor at the first record */ - return SELF_GOTOP( &pArea->area ); + return HB_SUCCESS; } /* @@ -1437,8 +1480,7 @@ static HB_ERRCODE hb_delimOpen( DELIMAREAP pArea, LPDBOPENINFO pOpenInfo ) { pArea->pFile = hb_fileExtOpen( szFileName, NULL, uiFlags | FXO_DEFAULTS | FXO_SHARELOCK | - FXO_COPYNAME | FXO_NOSEEKPOS, - NULL, pError ); + FXO_COPYNAME, NULL, pError ); if( ! pArea->pFile ) { if( ! pError ) @@ -1508,8 +1550,8 @@ static const RDDFUNCS delimTable = { NULL /* hb_delimBof */, NULL /* hb_delimEof */, NULL /* hb_delimFound */, NULL /* hb_delimGoBottom */, - NULL /* hb_delimGoTo */, - NULL /* hb_delimGoToId */, + ( DBENTRYP_UL ) hb_delimGoTo, + ( DBENTRYP_I ) hb_delimGoToId, ( DBENTRYP_V ) hb_delimGoTop, NULL /* hb_delimSeek */, NULL /* hb_delimSkip */, @@ -1606,12 +1648,6 @@ static const RDDFUNCS delimTable = { NULL /* hb_delimBof */, NULL /* hb_delimWhoCares */ }; - -/* - * -- DELIM METHODS -- - */ - - HB_FUNC( DELIM ) { ; } HB_FUNC_STATIC( DELIM_GETFUNCTABLE ) diff --git a/src/rdd/sdf1.c b/src/rdd/sdf1.c index 277af96693..076e3e012d 100644 --- a/src/rdd/sdf1.c +++ b/src/rdd/sdf1.c @@ -76,76 +76,107 @@ static void hb_sdfInitArea( SDFAREAP pArea, char * szFileName ) if( ! szEol || ! szEol[ 0 ] ) szEol = hb_conNewLine(); pArea->szEol = hb_strdup( szEol ); - pArea->uiEolLen = ( HB_USHORT ) strlen( pArea->szEol ); + pArea->uiEolLen = ( HB_USHORT ) strlen( szEol ); + pArea->fAnyEol = ( szEol[ 0 ] == '\n' || szEol[ 0 ] == '\r' ) && + ( pArea->uiEolLen == 1 || + ( pArea->uiEolLen == 2 && szEol[ 0 ] != szEol[ 1 ] && + ( szEol[ 1 ] == '\n' || szEol[ 1 ] == '\r' ) ) ); - /* Alloc buffer */ - pArea->pRecord = ( HB_BYTE * ) hb_xgrab( pArea->uiRecordLen + pArea->uiEolLen + 3 ); + /* allocate record buffer, one additional byte is for deleted flag */ + pArea->pRecord = ( HB_BYTE * ) hb_xgrab( pArea->uiRecordLen + pArea->uiEolLen + 1 ); /* pseudo deleted flag */ *pArea->pRecord++ = ' '; + memcpy( pArea->pRecord + pArea->uiRecordLen, + pArea->szEol, pArea->uiEolLen ); - pArea->nFileSize = 0; + if( pArea->fReadonly ) + { + /* allocate IO buffer */ + pArea->nBufferSize += pArea->fAnyEol ? 2 : pArea->uiEolLen; + if( pArea->nBufferSize < 8192 ) + pArea->nBufferSize = 8192; + pArea->pBuffer = ( HB_BYTE * ) hb_xgrab( pArea->nBufferSize ); + } pArea->ulRecCount = 0; + pArea->nBufferIndex = pArea->nBufferRead = pArea->nBufferSize; } static void hb_sdfClearRecordBuffer( SDFAREAP pArea ) { memset( pArea->pRecord, ' ', pArea->uiRecordLen ); - memcpy( pArea->pRecord + pArea->uiRecordLen, - pArea->szEol, pArea->uiEolLen ); } static HB_ERRCODE hb_sdfReadRecord( SDFAREAP pArea ) { - HB_USHORT uiRead, uiToRead, uiEolPos; + HB_SIZE nRead; HB_TRACE( HB_TR_DEBUG, ( "hb_sdfReadRecord(%p)", pArea ) ); - uiToRead = pArea->uiRecordLen + pArea->uiEolLen + 2; - uiRead = ( HB_USHORT ) hb_fileResult( - hb_fileReadAt( pArea->pFile, pArea->pRecord, uiToRead, - pArea->nRecordOffset ) ); - if( uiRead > 0 && uiRead < uiToRead && pArea->pRecord[ uiRead - 1 ] == '\032' ) - --uiRead; + pArea->area.fEof = HB_TRUE; - if( uiRead == 0 ) + nRead = 0; + for( ;; ) { - pArea->area.fEof = HB_TRUE; - pArea->fPositioned = HB_FALSE; - hb_sdfClearRecordBuffer( pArea ); + char ch; + + if( pArea->nBufferRead - pArea->nBufferIndex < ( HB_SIZE ) pArea->uiEolLen + 1 && + pArea->nBufferRead == pArea->nBufferSize ) + { + HB_SIZE nLeft = nRead >= ( HB_SIZE ) pArea->uiRecordLen ? 0 : + pArea->nBufferRead - pArea->nBufferIndex; + if( nLeft ) + memmove( pArea->pBuffer, + pArea->pBuffer + pArea->nBufferIndex, nLeft ); + pArea->nBufferIndex = 0; + pArea->nBufferRead = hb_fileRead( pArea->pFile, + pArea->pBuffer + nLeft, + pArea->nBufferSize - nLeft, -1 ); + if( pArea->nBufferRead == ( HB_SIZE ) FS_ERROR ) + pArea->nBufferRead = 0; + pArea->nBufferRead += nLeft; + } + + if( pArea->nBufferIndex >= pArea->nBufferRead ) + break; + + ch = pArea->pBuffer[ pArea->nBufferIndex++ ]; + + if( pArea->fAnyEol ) + { + if( ch == '\r' || ch == '\n' ) + { + if( pArea->nBufferIndex < pArea->nBufferRead && + pArea->pBuffer[ pArea->nBufferIndex ] != ch && + ( pArea->pBuffer[ pArea->nBufferIndex ] == '\r' || + pArea->pBuffer[ pArea->nBufferIndex ] == '\n' ) ) + pArea->nBufferIndex++; + pArea->area.fEof = HB_FALSE; + break; + } + } + else if( ch == pArea->szEol[ 0 ] ) + { + if( pArea->uiEolLen == 1 || + ( pArea->nBufferRead - pArea->nBufferIndex >= + ( HB_SIZE ) pArea->uiEolLen - 1 && + memcmp( pArea->pBuffer + pArea->nBufferIndex, + pArea->szEol + 1, pArea->uiEolLen - 1 ) == 0 ) ) + { + pArea->nBufferIndex += pArea->uiEolLen - 1; + pArea->area.fEof = HB_FALSE; + break; + } + } + if( nRead < ( HB_SIZE ) pArea->uiRecordLen && ch != '\032' ) + pArea->pRecord[ nRead++ ] = ch; } - else - { + + if( nRead < ( HB_SIZE ) pArea->uiRecordLen ) + memset( pArea->pRecord + nRead, ' ', pArea->uiRecordLen - nRead ); + if( nRead > 0 ) pArea->area.fEof = HB_FALSE; - pArea->fPositioned = HB_TRUE; - uiEolPos = ( HB_USHORT ) hb_strAt( pArea->szEol, pArea->uiEolLen, - ( const char * ) pArea->pRecord, - uiRead ); - if( uiEolPos ) - { - --uiEolPos; - if( uiRead < uiToRead && uiRead == uiEolPos + pArea->uiEolLen ) - pArea->nNextOffset = ( HB_FOFFSET ) -1; - else - pArea->nNextOffset = pArea->nRecordOffset + uiEolPos + pArea->uiEolLen; - if( uiEolPos < pArea->uiRecordLen ) - memset( pArea->pRecord + uiEolPos, ' ', pArea->uiRecordLen - uiEolPos ); - } - else - { - if( uiRead < uiToRead ) - pArea->nNextOffset = ( HB_FOFFSET ) -1; - else - pArea->nNextOffset = 0; - - if( uiRead < pArea->uiRecordLen ) - memset( pArea->pRecord + uiRead, ' ', pArea->uiRecordLen - uiRead ); - } - - if( uiEolPos != pArea->uiRecordLen ) - memcpy( pArea->pRecord + pArea->uiRecordLen, - pArea->szEol, pArea->uiEolLen ); - } + pArea->fPositioned = ! pArea->area.fEof; return HB_SUCCESS; } @@ -154,72 +185,55 @@ static HB_ERRCODE hb_sdfNextRecord( SDFAREAP pArea ) { HB_TRACE( HB_TR_DEBUG, ( "hb_sdfNextRecord(%p)", pArea ) ); - if( ! pArea->fPositioned ) - pArea->nNextOffset = ( HB_FOFFSET ) -1; - else + if( pArea->fPositioned ) { - if( pArea->nNextOffset == 0 ) - { - HB_USHORT uiRead, uiToRead, uiEolPos, uiRest = 0; - HB_FOFFSET ulOffset = pArea->nRecordOffset; - - uiToRead = pArea->uiRecordLen + pArea->uiEolLen + 2; - - do - { - uiRead = ( HB_USHORT ) hb_fileResult( - hb_fileReadAt( pArea->pFile, pArea->pRecord + uiRest, - uiToRead - uiRest, ulOffset + uiRest ) ) + uiRest; - if( uiRead > 0 && uiRead < uiToRead && - pArea->pRecord[ uiRead - 1 ] == '\032' ) - --uiRead; - - uiEolPos = ( HB_USHORT ) hb_strAt( pArea->szEol, pArea->uiEolLen, - ( const char * ) pArea->pRecord, - uiRead ); - if( uiEolPos ) - { - --uiEolPos; - if( uiRead < uiToRead && uiRead == uiEolPos + pArea->uiEolLen ) - pArea->nNextOffset = ( HB_FOFFSET ) -1; - else - pArea->nNextOffset = ulOffset + uiEolPos + pArea->uiEolLen; - } - else if( uiRead < uiToRead ) - { - pArea->nNextOffset = ( HB_FOFFSET ) -1; - } - else - { - if( pArea->uiEolLen > 1 ) - { - uiRest = pArea->uiEolLen - 1; - memcpy( pArea->pRecord, pArea->pRecord + uiRead - uiRest, uiRest ); - } - ulOffset += uiRead - uiRest; - } - } - while( pArea->nNextOffset == 0 ); - } pArea->ulRecNo++; + return hb_sdfReadRecord( pArea ); } - - if( pArea->nNextOffset == ( HB_FOFFSET ) -1 ) - { - pArea->area.fEof = HB_TRUE; - pArea->fPositioned = HB_FALSE; - hb_sdfClearRecordBuffer( pArea ); - return HB_SUCCESS; - } - - pArea->nRecordOffset = pArea->nNextOffset; - return hb_sdfReadRecord( pArea ); + return HB_SUCCESS; } /* * -- SDF METHODS -- */ +/* + * Position cursor at a specific physical record. + */ +static HB_ERRCODE hb_sdfGoTo( SDFAREAP pArea, HB_ULONG ulRecNo ) +{ + HB_TRACE( HB_TR_DEBUG, ( "hb_sdfGoTo(%p, %lu)", pArea, ulRecNo ) ); + +#ifndef HB_CLP_STRICT + if( pArea->fReadonly && ulRecNo >= pArea->ulRecNo ) + { + while( pArea->ulRecNo < ulRecNo && pArea->fPositioned ) + { + if( hb_sdfNextRecord( pArea ) != HB_SUCCESS ) + return HB_FAILURE; + } + return HB_SUCCESS; + } +#endif + /* generate RTE */ + return SUPER_GOTO( &pArea->area, ulRecNo ); +} + +/* + * Position the cursor to a specific, physical identity. + */ +static HB_ERRCODE hb_sdfGoToId( SDFAREAP pArea, PHB_ITEM pItem ) +{ + HB_TRACE( HB_TR_DEBUG, ( "hb_sdfGoToId(%p, %p)", pArea, pItem ) ); + +#ifndef HB_CLP_STRICT + if( HB_IS_NUMERIC( pItem ) ) + return SELF_GOTO( &pArea->area, hb_itemGetNL( pItem ) ); +#endif + /* generate RTE */ + return SUPER_GOTOID( &pArea->area, pItem ); +} + /* * Position cursor at the first record. */ @@ -233,10 +247,16 @@ static HB_ERRCODE hb_sdfGoTop( SDFAREAP pArea ) pArea->area.fTop = HB_TRUE; pArea->area.fBottom = HB_FALSE; - pArea->nRecordOffset = 0; - pArea->ulRecNo = 1; - if( hb_sdfReadRecord( pArea ) != HB_SUCCESS ) - return HB_FAILURE; + if( pArea->ulRecNo != 1 ) + { + if( pArea->ulRecNo != 0 || ! pArea->fReadonly ) + /* generate RTE */ + return SUPER_GOTOP( &pArea->area ); + + pArea->ulRecNo = 1; + if( hb_sdfReadRecord( pArea ) != HB_SUCCESS ) + return HB_FAILURE; + } return SELF_SKIPFILTER( &pArea->area, 1 ); } @@ -251,8 +271,9 @@ static HB_ERRCODE hb_sdfSkipRaw( SDFAREAP pArea, HB_LONG lToSkip ) if( SELF_GOCOLD( &pArea->area ) != HB_SUCCESS ) return HB_FAILURE; - if( lToSkip != 1 ) - return HB_FAILURE; + if( lToSkip != 1 || ! pArea->fReadonly ) + /* generate RTE */ + return SUPER_SKIPRAW( &pArea->area, lToSkip ); else return hb_sdfNextRecord( pArea ); } @@ -279,6 +300,7 @@ static HB_ERRCODE hb_sdfRecCount( SDFAREAP pArea, HB_ULONG * pRecCount ) HB_TRACE( HB_TR_DEBUG, ( "hb_sdfRecCount(%p,%p)", pArea, pRecCount ) ); *pRecCount = pArea->ulRecCount; + return HB_SUCCESS; } @@ -290,6 +312,7 @@ static HB_ERRCODE hb_sdfRecNo( SDFAREAP pArea, HB_ULONG * pulRecNo ) HB_TRACE( HB_TR_DEBUG, ( "hb_sdfRecNo(%p,%p)", pArea, pulRecNo ) ); *pulRecNo = pArea->ulRecNo; + return HB_SUCCESS; } @@ -337,7 +360,6 @@ static HB_ERRCODE hb_sdfAppend( SDFAREAP pArea, HB_BOOL fUnLockAll ) if( SELF_GOHOT( &pArea->area ) != HB_SUCCESS ) return HB_FAILURE; - pArea->nRecordOffset = pArea->nFileSize; pArea->ulRecNo = ++pArea->ulRecCount; pArea->area.fEof = HB_FALSE; pArea->fPositioned = HB_TRUE; @@ -427,7 +449,7 @@ static HB_ERRCODE hb_sdfGetValue( SDFAREAP pArea, HB_USHORT uiIndex, PHB_ITEM pI break; case HB_FT_DATE: - hb_itemPutDS( pItem, ( char * ) pArea->pRecord + pArea->pFieldOffset[ uiIndex ] ); + hb_itemPutDS( pItem, ( const char * ) pArea->pRecord + pArea->pFieldOffset[ uiIndex ] ); break; case HB_FT_TIMESTAMP: @@ -446,28 +468,22 @@ static HB_ERRCODE hb_sdfGetValue( SDFAREAP pArea, HB_USHORT uiIndex, PHB_ITEM pI case HB_FT_LONG: { HB_MAXINT lVal; - double dVal; - HB_BOOL fDbl; + double dVal; + HB_BOOL fDbl; fDbl = hb_strnToNum( ( const char * ) pArea->pRecord + pArea->pFieldOffset[ uiIndex ], pField->uiLen, &lVal, &dVal ); if( pField->uiDec ) - { hb_itemPutNDLen( pItem, fDbl ? dVal : ( double ) lVal, ( int ) ( pField->uiLen - pField->uiDec - 1 ), ( int ) pField->uiDec ); - } else if( fDbl ) - { hb_itemPutNDLen( pItem, dVal, ( int ) pField->uiLen, 0 ); - } else - { hb_itemPutNIntLen( pItem, lVal, ( int ) pField->uiLen ); - } + break; } - break; case HB_FT_MEMO: hb_itemPutC( pItem, NULL ); @@ -685,10 +701,9 @@ static HB_ERRCODE hb_sdfGoCold( SDFAREAP pArea ) if( pArea->fRecordChanged ) { - HB_SIZE nWrite = pArea->uiRecordLen + pArea->uiEolLen; + HB_SIZE nSize = pArea->uiRecordLen + pArea->uiEolLen; - if( hb_fileWriteAt( pArea->pFile, pArea->pRecord, nWrite, - pArea->nRecordOffset ) != nWrite ) + if( hb_fileWrite( pArea->pFile, pArea->pRecord, nSize, -1 ) != nSize ) { PHB_ITEM pError = hb_errNew(); @@ -701,8 +716,6 @@ static HB_ERRCODE hb_sdfGoCold( SDFAREAP pArea ) hb_itemRelease( pError ); return HB_FAILURE; } - pArea->nFileSize += nWrite; - pArea->nNextOffset = pArea->nFileSize; pArea->fRecordChanged = HB_FALSE; pArea->fFlush = HB_TRUE; } @@ -743,15 +756,10 @@ static HB_ERRCODE hb_sdfFlush( SDFAREAP pArea ) errCode = SELF_GOCOLD( &pArea->area ); - if( pArea->fFlush ) + if( pArea->fFlush && hb_setGetHardCommit() ) { - if( hb_setGetEOF() ) - hb_fileWriteAt( pArea->pFile, "\032", 1, pArea->nFileSize ); - if( hb_setGetHardCommit() ) - { - hb_fileCommit( pArea->pFile ); - pArea->fFlush = HB_FALSE; - } + hb_fileCommit( pArea->pFile ); + pArea->fFlush = HB_FALSE; } return errCode; @@ -960,6 +968,7 @@ static HB_ERRCODE hb_sdfNewArea( SDFAREAP pArea ) pArea->pFile = NULL; pArea->fTransRec = HB_TRUE; pArea->uiRecordLen = 0; + pArea->nBufferSize = 0; return HB_SUCCESS; } @@ -983,16 +992,23 @@ static HB_ERRCODE hb_sdfClose( SDFAREAP pArea ) { HB_TRACE( HB_TR_DEBUG, ( "hb_sdfClose(%p)", pArea ) ); - SUPER_CLOSE( &pArea->area ); - /* Update record and unlock records */ if( pArea->pFile ) { + SELF_GOCOLD( &pArea->area ); + + if( ! pArea->fReadonly && hb_setGetEOF() ) + { + hb_fileWrite( pArea->pFile, "\032", 1, -1 ); + pArea->fFlush = HB_TRUE; + } SELF_FLUSH( &pArea->area ); hb_fileClose( pArea->pFile ); pArea->pFile = NULL; } + SUPER_CLOSE( &pArea->area ); + if( pArea->pFieldOffset ) { hb_xfree( pArea->pFieldOffset ); @@ -1003,6 +1019,11 @@ static HB_ERRCODE hb_sdfClose( SDFAREAP pArea ) hb_xfree( pArea->pRecord - 1 ); pArea->pRecord = NULL; } + if( pArea->pBuffer ) + { + hb_xfree( pArea->pBuffer ); + pArea->pBuffer = NULL; + } if( pArea->szEol ) { hb_xfree( pArea->szEol ); @@ -1022,10 +1043,10 @@ static HB_ERRCODE hb_sdfClose( SDFAREAP pArea ) */ static HB_ERRCODE hb_sdfCreate( SDFAREAP pArea, LPDBOPENINFO pCreateInfo ) { - HB_ERRCODE errCode; - PHB_FNAME pFileName; PHB_ITEM pError = NULL; + HB_ERRCODE errCode; HB_BOOL fRetry; + PHB_FNAME pFileName; char szFileName[ HB_PATH_MAX ]; HB_TRACE( HB_TR_DEBUG, ( "hb_sdfCreate(%p,%p)", pArea, pCreateInfo ) ); @@ -1062,8 +1083,7 @@ static HB_ERRCODE hb_sdfCreate( SDFAREAP pArea, LPDBOPENINFO pCreateInfo ) { pArea->pFile = hb_fileExtOpen( szFileName, NULL, FO_READWRITE | FO_EXCLUSIVE | FXO_TRUNCATE | - FXO_DEFAULTS | FXO_SHARELOCK | FXO_COPYNAME | - FXO_NOSEEKPOS, + FXO_DEFAULTS | FXO_SHARELOCK | FXO_COPYNAME, NULL, pError ); if( ! pArea->pFile ) { @@ -1098,9 +1118,12 @@ static HB_ERRCODE hb_sdfCreate( SDFAREAP pArea, LPDBOPENINFO pCreateInfo ) } hb_sdfInitArea( pArea, szFileName ); + pArea->ulRecNo = 1; + pArea->area.fEof = HB_TRUE; + pArea->fPositioned = HB_FALSE; + hb_sdfClearRecordBuffer( pArea ); - /* Position cursor at the first record */ - return SELF_GOTOP( &pArea->area ); + return HB_SUCCESS; } /* @@ -1166,8 +1189,7 @@ static HB_ERRCODE hb_sdfOpen( SDFAREAP pArea, LPDBOPENINFO pOpenInfo ) { pArea->pFile = hb_fileExtOpen( szFileName, NULL, uiFlags | FXO_DEFAULTS | FXO_SHARELOCK | - FXO_COPYNAME | FXO_NOSEEKPOS, - NULL, pError ); + FXO_COPYNAME, NULL, pError ); if( ! pArea->pFile ) { if( ! pError ) @@ -1237,8 +1259,8 @@ static const RDDFUNCS sdfTable = { NULL /* hb_sdfBof */, NULL /* hb_sdfEof */, NULL /* hb_sdfFound */, NULL /* hb_sdfGoBottom */, - NULL /* hb_sdfGoTo */, - NULL /* hb_sdfGoToId */, + ( DBENTRYP_UL ) hb_sdfGoTo, + ( DBENTRYP_I ) hb_sdfGoToId, ( DBENTRYP_V ) hb_sdfGoTop, NULL /* hb_sdfSeek */, NULL /* hb_sdfSkip */, diff --git a/src/rtl/hbznet.c b/src/rtl/hbznet.c index 75350387bc..17ce9f1917 100644 --- a/src/rtl/hbznet.c +++ b/src/rtl/hbznet.c @@ -383,7 +383,7 @@ long hb_znetFlush( PHB_ZNETSTREAM pStream, HB_SOCKET sd, HB_MAXINT timeout, if( hb_znetStreamWrite( pStream, sd, timeout ) <= 0 ) break; - if( pStream->err == Z_OK ) + if( pStream->err == Z_OK || pStream->err == Z_BUF_ERROR ) pStream->err = deflate( &pStream->wr, fSync ? Z_FULL_FLUSH : Z_PARTIAL_FLUSH ); } diff --git a/src/rtl/hbzsock.c b/src/rtl/hbzsock.c index c67fffc4a6..e8ad931c97 100644 --- a/src/rtl/hbzsock.c +++ b/src/rtl/hbzsock.c @@ -211,7 +211,7 @@ static long s_sockexRead( PHB_SOCKEX pSock, void * data, long len, HB_MAXINT tim if( pZ->z_read.total_out != 0 ) lRecv = ( long ) pZ->z_read.total_out; - return pZ->z_read.total_out != 0 ? ( long ) pZ->z_read.total_out : lRecv; + return lRecv; } else return hb_sockexRead( pZ->sock, data, len, timeout ); @@ -277,7 +277,7 @@ static long s_sockexFlush( PHB_SOCKEX pSock, HB_MAXINT timeout, HB_BOOL fSync ) { if( s_zsock_write( pZ, timeout ) <= 0 ) break; - if( err == Z_OK ) + if( err == Z_OK || err == Z_BUF_ERROR ) err = deflate( &pZ->z_write, fSync ? Z_FULL_FLUSH : Z_PARTIAL_FLUSH ); } if( err != Z_OK && err != Z_BUF_ERROR )