fivedev/harbour-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2017-12-12 12:26 UTC+0100 Przemyslaw Czerpak (druzus/at/poczta.onet.pl)

Browse Source 
* contrib/hbnf/fttext.c
  * contrib/hbnf/hbnf.hbx
    * synced with Viktor's modifications:
         - use FILE API instead of FS API
         - protection against GPF when FT area has no valid file handle
         - alternative function names which exceeds 10 character Cl*pper limit
    ! fixed handle and memory leak when new file is open in FT area which
      already has open file

  * include/harbour.hbx
  * src/harbour.def
  * src/vm/hvm.c
    + added new PRG function: __vmItemRefs( <xVal> ) -> <nRefs>
      Please remember that passed <xVal> parameter increases the counter so
      if you want to eliminate it then pass it by reference, i,e.:
         aVal := {}
         ? __vmItemRefs( aVal ), __vmItemRefs( @aVal )

  * include/hbapi.h
  * src/harbour.def
  * src/vm/arrays.c
  * src/vm/codebloc.c
  * src/vm/hashes.c
    + added new C functions:
         HB_COUNTER hb_arrayRefs( PHB_ITEM pArray );
         HB_COUNTER hb_hashRefs( PHB_ITEM pHash );
         HB_COUNTER hb_codeblockRefs( PHB_ITEM pItem );

  * src/rtl/itemseri.c
    * added new internal function to replace 3 times repeated code for
      resizing array of references.
    % use new functions to check number of references to hashes, arrays and
      objects to ignore in system detecting multiple references the items
      which have reference counter smaller then 2. Such items cannot have
      multiple references.
      In practice this modification resolves the time problem when very big
      items are serialized to the level comparable to HB_SERIALIZE_IGNOREREF
      so now this flag is useless though I'll keep it because only in such
      mode serialization code can be used as filter in streams.
      I also created binary tree to store references but after this
      modifications it is not necessary so I decided to not commit it yet.
      Please only remember that practice limit of such serialization code
      is created by physical memory attached by OS to the process. When this
      limit is exceed then swap is actively used what completely kill the
      performance. The CPU usage is reduced to less then 1% because it waits
      for restoring swapped memory pages by extremely heavy used disks. In
      such case I cannot help.
This commit is contained in:
Przemysław Czerpak
2017-12-12 12:26:24 +01:00
parent f960a5ccda
commit 2b4c4b3b5f
11 changed files with 867 additions and 784 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/harbour.def
View File

@@ -1918,6 +1918,7 @@ HB_FUN___TRACEPRGCALLS
HB_FUN___TYPEFILE
HB_FUN___VMCOUNTTHREADS
HB_FUN___VMITEMID
HB_FUN___VMITEMREFS
HB_FUN___VMMODULESVERIFY
HB_FUN___VMNOINTERNALS
HB_FUN___WAIT
@@ -2031,6 +2032,7 @@ hb_arrayIsObject
hb_arrayLast
hb_arrayLen
hb_arrayNew
hb_arrayRefs
hb_arrayRevScan
hb_arrayScan
hb_arraySelfParams
@@ -2161,6 +2163,7 @@ hb_cmdargInit
hb_cmdargIsInternal
hb_cmdargProgName
hb_codeblockId
hb_codeblockRefs
hb_comClose
hb_comDiscardChar
hb_comErrorChar
@@ -2702,6 +2705,7 @@ hb_hashJoin
hb_hashLen
hb_hashNew
hb_hashPreallocate
hb_hashRefs
hb_hashRemove
hb_hashScan
hb_hashScanSoft

89
src/rtl/itemseri.c
View File

@@ -296,6 +296,33 @@ static PHB_REF_ITEM hb_itemSerialOffsetFind( PHB_REF_LIST pRefList, HB_SIZE nOff
return NULL;
}
static PHB_REF_ITEM hb_itemSerialRefNew( PHB_REF_LIST pRefList, HB_SIZE nPos )
{
PHB_REF_ITEM pRef;
HB_SIZE nMove;
if( pRefList->nCount >= pRefList->nSize )
{
if( pRefList->nSize == 0 )
pRefList->nSize = HB_SERIAL_REFLSTINIT;
else
pRefList->nSize += pRefList->nSize >> 1;
pRefList->pRefs = ( PHB_REF_ITEM )
hb_xrealloc( pRefList->pRefs,
pRefList->nSize * sizeof( HB_REF_ITEM ) );
}
nMove = pRefList->nCount - nPos;
pRef = &pRefList->pRefs[ pRefList->nCount++ ];
while( nMove-- > 0 )
{
*pRef = *( pRef - 1 );
pRef--;
}
return pRef;
}
/* used by hb_itemSerialSize() for HB_IT_ARRAY and HB_IT_HASH */
static HB_BOOL hb_itemSerialValueRef( PHB_REF_LIST pRefList, void * value,
HB_SIZE nOffset )
@@ -309,21 +336,7 @@ static HB_BOOL hb_itemSerialValueRef( PHB_REF_LIST pRefList, void * value,
return HB_TRUE;
}
if( pRefList->nCount >= pRefList->nSize )
{
if( pRefList->nSize == 0 )
pRefList->nSize = HB_SERIAL_REFLSTINIT;
else
pRefList->nSize += pRefList->nSize >> 1;
pRefList->pRefs = ( PHB_REF_ITEM )
hb_xrealloc( pRefList->pRefs,
pRefList->nSize * sizeof( HB_REF_ITEM ) );
}
pRef = &pRefList->pRefs[ nPos ];
if( nPos < pRefList->nCount )
memmove( pRef + 1, pRef, ( pRefList->nCount - nPos ) * sizeof( HB_REF_ITEM ) );
pRefList->nCount++;
pRef = hb_itemSerialRefNew( pRefList, nPos );
pRef->value = value;
pRef->nOffset = nOffset;
@@ -384,21 +397,7 @@ static HB_BOOL hb_itemSerialOffsetRef( PHB_REF_LIST pRefList, HB_SIZE nOffset )
if( hb_itemSerialOffsetFind( pRefList, nOffset, 0, &nPos ) != NULL )
return HB_TRUE;
if( pRefList->nCount >= pRefList->nSize )
{
if( pRefList->nSize == 0 )
pRefList->nSize = HB_SERIAL_REFLSTINIT;
else
pRefList->nSize += pRefList->nSize >> 1;
pRefList->pRefs = ( PHB_REF_ITEM )
hb_xrealloc( pRefList->pRefs,
pRefList->nSize * sizeof( HB_REF_ITEM ) );
}
pRef = &pRefList->pRefs[ nPos ];
if( nPos < pRefList->nCount )
memmove( pRef + 1, pRef, ( pRefList->nCount - nPos ) * sizeof( HB_REF_ITEM ) );
pRefList->nCount++;
pRef = hb_itemSerialRefNew( pRefList, nPos );
pRef->value = NULL;
pRef->nOffset = nOffset;
@@ -416,23 +415,7 @@ static void hb_itemSerialTypedRef( PHB_REF_LIST pRefList, int iType,
if( hb_itemSerialOffsetFind( pRefList, nIndex, iType, &nPos ) == NULL )
{
PHB_REF_ITEM pRef;
if( pRefList->nCount >= pRefList->nSize )
{
if( pRefList->nSize == 0 )
pRefList->nSize = HB_SERIAL_REFLSTINIT;
else
pRefList->nSize += pRefList->nSize >> 1;
pRefList->pRefs = ( PHB_REF_ITEM )
hb_xrealloc( pRefList->pRefs,
pRefList->nSize * sizeof( HB_REF_ITEM ) );
}
pRef = &pRefList->pRefs[ nPos ];
if( nPos < pRefList->nCount )
memmove( pRef + 1, pRef, ( pRefList->nCount - nPos ) * sizeof( HB_REF_ITEM ) );
pRefList->nCount++;
PHB_REF_ITEM pRef = hb_itemSerialRefNew( pRefList, nPos );
pRef->value = NULL;
pRef->nOffset = nIndex;
@@ -585,7 +568,7 @@ static HB_SIZE hb_itemSerialSize( PHB_ITEM pItem, int iFlags,
if( szClass && szFunc )
nSize += strlen( szClass ) + strlen( szFunc ) + 3;
}
if( ( iFlags & HB_SERIALIZE_IGNOREREF ) == 0 &&
if( ( iFlags & HB_SERIALIZE_IGNOREREF ) == 0 && hb_arrayRefs( pItem ) > 1 &&
hb_itemSerialValueRef( pRefList, hb_arrayId( pItem ), nOffset + nSize ) )
{
nSize = 5;
@@ -606,7 +589,7 @@ static HB_SIZE hb_itemSerialSize( PHB_ITEM pItem, int iFlags,
break;
case HB_IT_HASH:
if( ( iFlags & HB_SERIALIZE_IGNOREREF ) == 0 &&
if( ( iFlags & HB_SERIALIZE_IGNOREREF ) == 0 && hb_hashRefs( pItem ) > 1 &&
hb_itemSerialValueRef( pRefList, hb_hashId( pItem ), nOffset ) )
{
nSize = 5;
@@ -875,7 +858,9 @@ static HB_SIZE hb_serializeItem( PHB_ITEM pItem, HB_BOOL iFlags,
break;
case HB_IT_ARRAY:
if( hb_itemSerialValueOffset( pRefList, hb_arrayId( pItem ), nOffset, &nRef ) )
nRef = HB_SERIAL_DUMMYOFFSET;
if( hb_arrayRefs( pItem ) > 1 &&
hb_itemSerialValueOffset( pRefList, hb_arrayId( pItem ), nOffset, &nRef ) )
{
pBuffer[ nOffset++ ] = HB_SERIAL_REF;
HB_PUT_LE_UINT32( &pBuffer[ nOffset ], nRef );
@@ -927,7 +912,9 @@ static HB_SIZE hb_serializeItem( PHB_ITEM pItem, HB_BOOL iFlags,
break;
case HB_IT_HASH:
if( hb_itemSerialValueOffset( pRefList, hb_hashId( pItem ), nOffset, &nRef ) )
nRef = HB_SERIAL_DUMMYOFFSET;
if( hb_hashRefs( pItem ) > 1 &&
hb_itemSerialValueOffset( pRefList, hb_hashId( pItem ), nOffset, &nRef ) )
{
pBuffer[ nOffset++ ] = HB_SERIAL_REF;
HB_PUT_LE_UINT32( &pBuffer[ nOffset ], nRef );

9
src/vm/arrays.c
View File

@@ -300,6 +300,15 @@ void * hb_arrayId( PHB_ITEM pArray )
return NULL;
}
/* retrieves numer of references to the array */
HB_COUNTER hb_arrayRefs( PHB_ITEM pArray )
{
if( pArray && HB_IS_ARRAY( pArray ) )
return hb_gcRefCount( pArray->item.asArray.value );
else
return 0;
}
PHB_ITEM hb_arrayFromId( PHB_ITEM pItem, void * pArrayId )
{
HB_STACK_TLS_PRELOAD

9
src/vm/codebloc.c
View File

@@ -293,3 +293,12 @@ void * hb_codeblockId( PHB_ITEM pItem )
else
return NULL;
}
/* retrieves numer of references to the codeblock */
HB_COUNTER hb_codeblockRefs( PHB_ITEM pItem )
{
if( HB_IS_BLOCK( pItem ) )
return hb_gcRefCount( pItem->item.asBlock.value );
else
return 0;
}

9
src/vm/hashes.c
View File

@@ -946,6 +946,15 @@ void * hb_hashId( PHB_ITEM pHash )
return NULL;
}
/* retrieves numer of references to the hash */
HB_COUNTER hb_hashRefs( PHB_ITEM pHash )
{
if( HB_IS_HASH( pHash ) )
return hb_gcRefCount( pHash->item.asHash.value );
else
return 0;
}
void hb_hashCloneBody( PHB_ITEM pDest, PHB_ITEM pHash, PHB_NESTED_CLONED pClonedList )
{
HB_SIZE nPos;

21
src/vm/hvm.c
View File

@@ -12421,6 +12421,27 @@ HB_FUNC( __VMITEMID )
}
}
HB_FUNC( __VMITEMREFS )
{
HB_STACK_TLS_PRELOAD
PHB_ITEM pItem = hb_param( 1, HB_IT_ANY );
if( pItem )
{
if( HB_IS_ARRAY( pItem ) )
hb_retnint( hb_arrayRefs( pItem ) );
else if( HB_IS_HASH( pItem ) )
hb_retnint( hb_hashRefs( pItem ) );
else if( HB_IS_BLOCK( pItem ) )
hb_retnint( hb_codeblockRefs( pItem ) );
else if( HB_IS_POINTER( pItem ) )
hb_retnint( hb_gcRefCount( pItem->item.asPointer.value ) );
else if( HB_IS_STRING( pItem ) )
hb_retnint( hb_xRefCount( pItem->item.asString.value ) );
}
}
HB_FUNC( __VMMODULESVERIFY )
{
HB_STACK_TLS_PRELOAD