From 04283dd94fe408adff6f39ccd36cf192988f15e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Czerpak?= Date: Thu, 14 Jan 2016 11:02:51 +0100 Subject: [PATCH] 2016-01-14 11:02 UTC+0100 Przemyslaw Czerpak (druzus/at/poczta.onet.pl) * config/global.mk * set HARBOUR=F:100 during build process in OS2 builds - it increases maximum number of open handles per process from 20 to 100 so it should be possible to use -j switch during Harbour compilation - please test, i.e.: os2-make -j3 * contrib/xhb/hbcompat.ch ! fixed pp rules for Str(,,, ) translation * include/hbapifs.h * src/rtl/filesys.c + added new C function: HB_FHANDLE hb_fsOpenEx( const char * pszFileName, HB_FATTR nAttr, HB_USHORT uiFlags ); * minor cleanup * include/hbmemory.ch * src/vm/fm.c + added two new actions for Memory() function: HB_MEM_STATISTICS - return non 0 value if FM statistic is enabled HB_MEM_ISLIMIT - return non 0 value if used memory limit is supported + added new PRG function: __fm_allocLimit( [ ] ) -> It allows to set limit for maximum used memory in some memory managers, -1 means no limit. Now it works with DLMALLOC and in ST mode it's limit for memory allocated from system for whole application. In MT mode when HB_FM_DLMT_ALLOC (default for DLMALLOC) is enabled it's limit for single MSPACE (Harbour allocates 16 MSPACES and balance them between threads to improve scalability) so it's rather per thread limit. If memory statistic module is enabled during Harbour build process then this limit works with any memory manager and defines limit for total memory allocated by Harbour application regardles of MT or ST mode. * src/vm/hvm.c * use HB_MEM_STATISTICS instead of HB_MEM_USEDMAX to check if memory statistic module is enabled --- ChangeLog.txt | 42 +++++++++++++++++++ config/global.mk | 5 +++ contrib/xhb/hbcompat.ch | 4 +- include/hbapifs.h | 1 + include/hbmemory.ch | 3 +- src/rtl/filesys.c | 20 +++++---- src/vm/fm.c | 90 +++++++++++++++++++++++++++++++++++++++-- src/vm/hvm.c | 2 +- 8 files changed, 153 insertions(+), 14 deletions(-) diff --git a/ChangeLog.txt b/ChangeLog.txt index dbce88be02..11c07e995c 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -10,6 +10,48 @@ * Change, ! Fix, % Optimization, + Addition, - Removal, ; Comment */ +2016-01-14 11:02 UTC+0100 Przemyslaw Czerpak (druzus/at/poczta.onet.pl) + * config/global.mk + * set HARBOUR=F:100 during build process in OS2 builds - it increases + maximum number of open handles per process from 20 to 100 so it should + be possible to use -j switch during Harbour compilation - please + test, i.e.: + os2-make -j3 + + * contrib/xhb/hbcompat.ch + ! fixed pp rules for Str(,,, ) translation + + * include/hbapifs.h + * src/rtl/filesys.c + + added new C function: + HB_FHANDLE hb_fsOpenEx( const char * pszFileName, HB_FATTR nAttr, + HB_USHORT uiFlags ); + * minor cleanup + + * include/hbmemory.ch + * src/vm/fm.c + + added two new actions for Memory() function: + HB_MEM_STATISTICS - return non 0 value if FM statistic is enabled + HB_MEM_ISLIMIT - return non 0 value if used memory limit is + supported + + added new PRG function: + __fm_allocLimit( [ ] ) -> + It allows to set limit for maximum used memory in some memory + managers, -1 means no limit. + Now it works with DLMALLOC and in ST mode it's limit for memory + allocated from system for whole application. In MT mode when + HB_FM_DLMT_ALLOC (default for DLMALLOC) is enabled it's limit for + single MSPACE (Harbour allocates 16 MSPACES and balance them + between threads to improve scalability) so it's rather per thread + limit. If memory statistic module is enabled during Harbour build + process then this limit works with any memory manager and defines + limit for total memory allocated by Harbour application regardles + of MT or ST mode. + + * src/vm/hvm.c + * use HB_MEM_STATISTICS instead of HB_MEM_USEDMAX to check if memory + statistic module is enabled + 2016-01-13 15:25 UTC+0100 Przemyslaw Czerpak (druzus/at/poczta.onet.pl) * contrib/hbtip/encurlc.c ! fixed C&P typos in tip_URLEncode() and tip_URLDecode() in may diff --git a/config/global.mk b/config/global.mk index a1d0e65523..2f0fe2b19f 100644 --- a/config/global.mk +++ b/config/global.mk @@ -2042,6 +2042,11 @@ export HARBOURCMD := export CLIPPER := export CLIPPERCMD := +# in OS2 increase maximum number of file handle from 20 to 100 for -j build switch +ifeq ($(HB_HOST_PLAT),os2) + export HARBOUR := F:100 +endif + # relevant only on non-*nix hosts where --print-directory is on by default ifeq ($(findstring w,$(MAKEFLAGS)),) MKFLAGS := --no-print-directory diff --git a/contrib/xhb/hbcompat.ch b/contrib/xhb/hbcompat.ch index 9472df0cb4..1e17cdeb6b 100644 --- a/contrib/xhb/hbcompat.ch +++ b/contrib/xhb/hbcompat.ch @@ -274,7 +274,9 @@ #xtranslate MaxCol( .T. ) => hb_gtInfo( HB_GTI_VIEWPORTWIDTH ) #xtranslate NextKey( [] ) => hb_keyNext( ) - #xtranslate Str( , [], [], ) => iif( , hb_ntos( ), Str( ) ) + #xtranslate Str( , , , ) => iif( , LTrim( Str( , , ) ), Str( , , ) ) + #xtranslate Str( , ,, ) => iif( , LTrim( Str( , ) ), Str( , ) ) + #xtranslate Str( ,,, ) => iif( , hb_ntos( ), Str( ) ) #xuntranslate NetName( => #xuntranslate MemoWrit( => diff --git a/include/hbapifs.h b/include/hbapifs.h index ea8cf0a86c..3152538456 100644 --- a/include/hbapifs.h +++ b/include/hbapifs.h @@ -153,6 +153,7 @@ extern HB_EXPORT int hb_fsLockTest ( HB_FHANDLE hFileHandle, HB_FOFFSE HB_FOFFSET nLength, HB_USHORT uiMode ); extern HB_EXPORT HB_BOOL hb_fsMkDir ( const char * pszDirName ); /* create a directory */ extern HB_EXPORT HB_FHANDLE hb_fsOpen ( const char * pszFileName, HB_USHORT uiFlags ); /* open a file */ +extern HB_EXPORT HB_FHANDLE hb_fsOpenEx ( const char * pszFileName, HB_FATTR nAttr, HB_USHORT uiFlags ); /* open or new create a file with given attributes */ extern HB_EXPORT HB_USHORT hb_fsRead ( HB_FHANDLE hFileHandle, void * pBuff, HB_USHORT uiCount ); /* read contents of a file into a buffer (<=64K) */ extern HB_EXPORT HB_SIZE hb_fsReadLarge ( HB_FHANDLE hFileHandle, void * pBuff, HB_SIZE nCount ); /* read contents of a file into a buffer (>64K) */ extern HB_EXPORT HB_SIZE hb_fsReadAt ( HB_FHANDLE hFileHandle, void * pBuff, HB_SIZE nCount, HB_FOFFSET nOffset ); /* read from given offset contents of a file into a buffer (>64K) */ diff --git a/include/hbmemory.ch b/include/hbmemory.ch index f618323a0d..dcfef3d58c 100644 --- a/include/hbmemory.ch +++ b/include/hbmemory.ch @@ -74,5 +74,6 @@ #define HB_MEM_STACK 1004 /* Total memory size used by the stack (bytes) */ #define HB_MEM_STACK_TOP 1005 /* Total items currently on the stack */ #define HB_MEM_BLOCKS 1007 /* Total number of memory blocks allocated */ - +#define HB_MEM_STATISTICS 1008 /* Return non 0 value if FM statistic is enabled */ +#define HB_MEM_ISLIMIT 1009 /* Return non 0 value if used memory limit is supported */ #endif /* HB_MEMORY_CH_ */ diff --git a/src/rtl/filesys.c b/src/rtl/filesys.c index 5a83227876..037e8de35b 100644 --- a/src/rtl/filesys.c +++ b/src/rtl/filesys.c @@ -608,7 +608,7 @@ static void convert_open_flags( HB_BOOL fCreate, HB_FATTR nAttr, HB_USHORT uiFla #endif /* dos file attributes */ -#if defined( HB_FS_DOSATTR ) +#if defined( HB_FS_DOSCREAT ) if( nAttr == FC_NORMAL ) { *attr = _A_NORMAL; @@ -634,7 +634,6 @@ static void convert_open_flags( HB_BOOL fCreate, HB_FATTR nAttr, HB_USHORT uiFla } else { - *attr = 0; *flags = O_BINARY | O_LARGEFILE; switch( uiFlags & ( FO_READ | FO_WRITE | FO_READWRITE ) ) { @@ -1429,10 +1428,17 @@ HB_SIZE hb_fsPipeWrite( HB_FHANDLE hPipeHandle, const void * buffer, HB_SIZE nSi } HB_FHANDLE hb_fsOpen( const char * pszFileName, HB_USHORT uiFlags ) +{ + HB_TRACE( HB_TR_DEBUG, ( "hb_fsOpen(%s, %hu)", pszFileName, uiFlags ) ); + + return hb_fsOpenEx( pszFileName, FC_NORMAL, uiFlags ); +} + +HB_FHANDLE hb_fsOpenEx( const char * pszFileName, HB_FATTR nAttr, HB_USHORT uiFlags ) { HB_FHANDLE hFileHandle; - HB_TRACE( HB_TR_DEBUG, ( "hb_fsOpen(%s, %hu)", pszFileName, uiFlags ) ); + HB_TRACE( HB_TR_DEBUG, ( "hb_fsOpenEx(%s, %u, %hu)", pszFileName, nAttr, uiFlags ) ); #if defined( HB_OS_WIN ) { @@ -1443,7 +1449,7 @@ HB_FHANDLE hb_fsOpen( const char * pszFileName, HB_USHORT uiFlags ) lpFileName = HB_FSNAMECONV( pszFileName, &lpFree ); - convert_open_flags( HB_FALSE, FC_NORMAL, uiFlags, &dwMode, &dwShare, &dwCreat, &dwAttr ); + convert_open_flags( HB_FALSE, nAttr, uiFlags, &dwMode, &dwShare, &dwCreat, &dwAttr ); hb_vmUnlock(); hFile = CreateFile( lpFileName, dwMode, dwShare, NULL, dwCreat, dwAttr, NULL ); @@ -1459,7 +1465,7 @@ HB_FHANDLE hb_fsOpen( const char * pszFileName, HB_USHORT uiFlags ) { ULONG ulAction = 0, ulAttribute, fsOpenFlags, fsOpenMode; - convert_open_flags( HB_FALSE, FC_NORMAL, uiFlags, + convert_open_flags( HB_FALSE, nAttr, uiFlags, &ulAttribute, &fsOpenFlags, &fsOpenMode ); hb_vmUnlock(); hb_fsOS2DosOpenL( pszFileName, &hFileHandle, &ulAction, 0, @@ -1474,7 +1480,7 @@ HB_FHANDLE hb_fsOpen( const char * pszFileName, HB_USHORT uiFlags ) pszFileName = hb_fsNameConv( pszFileName, &pszFree ); - convert_open_flags( HB_FALSE, FC_NORMAL, uiFlags, &flags, &mode, &share, &attr ); + convert_open_flags( HB_FALSE, nAttr, uiFlags, &flags, &mode, &share, &attr ); hb_vmUnlock(); #if defined( _MSC_VER ) || defined( __DMC__ ) @@ -4544,7 +4550,7 @@ HB_FHANDLE hb_fsExtOpen( const char * pszFileName, const char * pDefExt, uiFlags |= FO_TRUNC; } - hFile = hb_fsOpen( szPath, uiFlags ); + hFile = hb_fsOpenEx( szPath, FC_NORMAL, uiFlags ); #if defined( HB_USE_SHARELOCKS ) if( hFile != FS_ERROR && ( nExFlags & FXO_SHARELOCK ) != 0 ) diff --git a/src/vm/fm.c b/src/vm/fm.c index 6156732ea5..0d7ce4f6d1 100644 --- a/src/vm/fm.c +++ b/src/vm/fm.c @@ -334,8 +334,9 @@ static HB_BOOL s_fStatistic = HB_FALSE; static HB_ISIZ s_nMemoryBlocks = 0; /* memory blocks used */ static HB_ISIZ s_nMemoryMaxBlocks = 0; /* maximum number of used memory blocks */ -static HB_ISIZ s_nMemoryMaxConsumed = 0; /* memory size consumed */ -static HB_ISIZ s_nMemoryConsumed = 0; /* memory max size consumed */ +static HB_ISIZ s_nMemoryConsumed = 0; /* memory size consumed */ +static HB_ISIZ s_nMemoryMaxConsumed = 0; /* memory max size consumed */ +static HB_ISIZ s_nMemoryLimConsumed = 0; /* limit the size of memory consumed */ static PHB_MEMINFO s_pFirstBlock = NULL; static PHB_MEMINFO s_pLastBlock = NULL; @@ -666,6 +667,12 @@ void * hb_xalloc( HB_SIZE nSize ) /* allocates fixed memory, returns NUL HB_FM_UNLOCK(); + if( s_nMemoryLimConsumed > 0 && s_nMemoryConsumed > s_nMemoryLimConsumed ) + { + free( pMem ); + return NULL; + } + #ifdef HB_PARANOID_MEM_CHECK memset( HB_MEM_PTR( pMem ), HB_MEMFILER, nSize ); #endif @@ -751,10 +758,15 @@ void * hb_xgrab( HB_SIZE nSize ) /* allocates fixed memory, exits on fai HB_FM_UNLOCK(); + if( s_nMemoryLimConsumed > 0 && s_nMemoryConsumed > s_nMemoryLimConsumed ) + { + s_nMemoryLimConsumed = 0; + hb_errInternal( HB_EI_XGRABALLOC, NULL, NULL, NULL ); + } + #ifdef HB_PARANOID_MEM_CHECK memset( HB_MEM_PTR( pMem ), HB_MEMFILER, nSize ); #endif - } #endif /* HB_FM_STATISTICS */ @@ -845,6 +857,12 @@ void * hb_xrealloc( void * pMem, HB_SIZE nSize ) /* reallocates memory */ HB_FM_UNLOCK(); + if( s_nMemoryLimConsumed > 0 && s_nMemoryConsumed > s_nMemoryLimConsumed ) + { + s_nMemoryLimConsumed = 0; + hb_errInternal( HB_EI_XREALLOC, NULL, NULL, NULL ); + } + #if defined( HB_PARANOID_MEM_CHECK ) || defined( HB_FM_FORCE_REALLOC ) # ifdef HB_PARANOID_MEM_CHECK memset( pMemBlock, HB_MEMFILER, HB_ALLOC_SIZE( nMemSize ) ); @@ -943,7 +961,7 @@ void hb_xfree( void * pMem ) /* frees fixed memory */ #endif } - free( ( void * ) pMemBlock ); + free( pMemBlock ); #else @@ -1449,6 +1467,10 @@ HB_SIZE hb_xquery( int iMode ) case HB_MEM_USED: /* Harbour extension (Memory used [bytes]) */ #ifdef HB_FM_STATISTICS nResult = s_nMemoryConsumed; +#elif defined( HB_FM_DLMT_ALLOC ) + nResult = mspace_footprint( hb_mspace() ); +#elif defined( HB_FM_DL_ALLOC ) + nResult = dlmalloc_footprint(); #else nResult = 0; #endif @@ -1465,6 +1487,10 @@ HB_SIZE hb_xquery( int iMode ) case HB_MEM_USEDMAX: /* Harbour extension (Maximum memory used [bytes]) */ #ifdef HB_FM_STATISTICS nResult = s_nMemoryMaxConsumed; +#elif defined( HB_FM_DLMT_ALLOC ) + nResult = mspace_max_footprint( hb_mspace() ); +#elif defined( HB_FM_DL_ALLOC ) + nResult = dlmalloc_max_footprint(); #else nResult = 0; #endif @@ -1484,6 +1510,26 @@ HB_SIZE hb_xquery( int iMode ) nResult = hb_stackTopOffset(); break; } + case HB_MEM_STATISTICS: /* Harbour extension (Is FM statistic is enabled?) */ +#ifdef HB_FM_STATISTICS + nResult = s_fStatistic; +#else + nResult = 0; +#endif + break; + + case HB_MEM_ISLIMIT: /* Harbour extension (Is used memory limit supported?) */ +#if defined( HB_FM_DLMT_ALLOC ) + nResult = 1; +#elif defined( HB_FM_DL_ALLOC ) + nResult = 1; +#elif defined( HB_FM_STATISTICS ) + nResult = s_fStatistic; +#else + nResult = 0; +#endif + break; + default: nResult = 0; } @@ -1499,3 +1545,39 @@ HB_BOOL hb_xtraced( void ) return HB_FALSE; #endif } + +HB_FUNC( __FM_ALLOCLIMIT ) +{ + hb_xclean(); +#if defined( HB_FM_DLMT_ALLOC ) + hb_retns( mspace_footprint_limit( hb_mspace() ) ); + if( HB_ISNUM( 1 ) ) + { + HB_ISIZE nLimit = hb_parns( 1 ); + + if( nLimit <= 0 ) + nLimit = -1; + mspace_set_footprint_limit( hb_mspace(), nLimit ); + } +#elif defined( HB_FM_DL_ALLOC ) + hb_retns( dlmalloc_footprint_limit() ); + if( HB_ISNUM( 1 ) ) + { + HB_ISIZE nLimit = hb_parns( 1 ); + + if( nLimit <= 0 ) + nLimit = -1; + dlmalloc_set_footprint_limit( ( size_t ) nLimit ); + } +#elif defined( HB_FM_STATISTICS ) + hb_retns( s_nMemoryLimConsumed ? s_nMemoryLimConsumed : -1 ); + if( HB_ISNUM( 1 ) ) + { + HB_ISIZE nLimit = hb_parns( 1 ); + + s_nMemoryLimConsumed = HB_MAX( nLimit, 0 ); + } +#else + hb_retni( 0 ); +#endif +} diff --git a/src/vm/hvm.c b/src/vm/hvm.c index 3c5999e80a..006f1fb582 100644 --- a/src/vm/hvm.c +++ b/src/vm/hvm.c @@ -1303,7 +1303,7 @@ int hb_vmQuit( void ) hb_cdpReleaseAll(); /* releases codepages */ /* release all known garbage */ - if( hb_xquery( HB_MEM_USEDMAX ) == 0 ) /* check if fmstat is ON */ + if( hb_xquery( HB_MEM_STATISTICS ) == 0 ) /* check if fmstat is ON */ hb_gcReleaseAll(); hb_vmUnsetExceptionHandler();