2019-02-11 13:09 UTC+0100 Aleksander Czajczynski (hb fki.pl)

* contrib/sddsqlt3/core.c
    ! fix DBUSEAREA() operation with SQLITE3 SDD to return empty result
      when query conditions are false or the source table has no rows.

      Previously an logically correct example caused RTE:
      DBUSEAREA(,, "SELECT * FROM existing_table WHERE FALSE")

    ! fix double-free error in sqLite3Disconnect(), looks like the
      sqlite3_close() return value checking was reverted, SQLITE_OK is 0

    * use CDP API to get UTF8 string length

    * use new sqlite3_prepare_v3() when built against
      sqlite 3.20.0 or upper (change borrowed from Viktor's 3.4 fork)

    + add HB_SQLT3_MAP_DECLARED_EMULATED define (not yet enabled by default)
      which make this SDD additionally parse SQLite column declarations.
      Right now it can make HB_FT_DATE fields working using standard
      ISO 8601 "yyyy-mm-dd" syntax. Also declarations not significant for
      SQLite, but useful in xBase-style programming - SQL numeric(len,dec)
      columns are detected in this mode and will be reflected in dbStruct().

    + add support for alternative StoD() like syntax for HB_FT_DATE columns

    + add support for ISO 8601 "YYYY-MM-DD HH:MM:SS.FFF" timestamp declared
      columns, SQLite stored strings are converted to proper HB_FT_TIMESTAMP
      fields

    + added HB_SQLT3_FIELDNAME_STRICT define, which enables shortening
      of field to "name" if SQLite returns "table.name". Such fields are
      not completly usable in xBase code - WA->T.FIELD syntax is not
      valid, but FieldPos("t.field") is OK. I think it should be default
      behaviour or some runtime setting should be introduced for convenience
      when working with specific SQL queries.

  * contrib/rddsql/sqlbase.c
  * contrib/rddsql/sqlmix.c
    + added ZAP functionality to SQLBASE and SQLMIX RDDs,
      index tags are preserved while ZAP-ing SQLMIX area.
      They are cleaned, no REINDEX is needed

    * changed to allow values of any type in "V" SIX3 / HB_FT_ANY fields
      in SQLBASE/SQLMIX RDD workareas

  * contrib/hbfoxpro/relfunc.c
    ! fix InList() FoxPro compatible function not looking at the last
      parameter passed. Thanks to Attila Szabo for the information
      posted on the developers list.

ChangeLog.txt

@@ -7,6 +7,56 @@
    Entries may not always be in chronological/commit order.
    See license at the end of file. */
 
+2019-02-11 13:09 UTC+0100 Aleksander Czajczynski (hb fki.pl)
+  * contrib/sddsqlt3/core.c
+    ! fix DBUSEAREA() operation with SQLITE3 SDD to return empty result
+      when query conditions are false or the source table has no rows.
+
+      Previously an logically correct example caused RTE:
+      DBUSEAREA(,, "SELECT * FROM existing_table WHERE FALSE")
+
+    ! fix double-free error in sqLite3Disconnect(), looks like the
+      sqlite3_close() return value checking was reverted, SQLITE_OK is 0
+
+    * use CDP API to get UTF8 string length
+
+    * use new sqlite3_prepare_v3() when built against
+      sqlite 3.20.0 or upper (change borrowed from Viktor's 3.4 fork)
+
+    + add HB_SQLT3_MAP_DECLARED_EMULATED define (not yet enabled by default)
+      which make this SDD additionally parse SQLite column declarations.
+      Right now it can make HB_FT_DATE fields working using standard
+      ISO 8601 "yyyy-mm-dd" syntax. Also declarations not significant for
+      SQLite, but useful in xBase-style programming - SQL numeric(len,dec)
+      columns are detected in this mode and will be reflected in dbStruct().
+
+    + add support for alternative StoD() like syntax for HB_FT_DATE columns
+
+    + add support for ISO 8601 "YYYY-MM-DD HH:MM:SS.FFF" timestamp declared
+      columns, SQLite stored strings are converted to proper HB_FT_TIMESTAMP
+      fields
+
+    + added HB_SQLT3_FIELDNAME_STRICT define, which enables shortening
+      of field to "name" if SQLite returns "table.name". Such fields are
+      not completly usable in xBase code - WA->T.FIELD syntax is not
+      valid, but FieldPos("t.field") is OK. I think it should be default
+      behaviour or some runtime setting should be introduced for convenience
+      when working with specific SQL queries.
+
+  * contrib/rddsql/sqlbase.c
+  * contrib/rddsql/sqlmix.c
+    + added ZAP functionality to SQLBASE and SQLMIX RDDs,
+      index tags are preserved while ZAP-ing SQLMIX area.
+      They are cleaned, no REINDEX is needed
+
+    * changed to allow values of any type in "V" SIX3 / HB_FT_ANY fields
+      in SQLBASE/SQLMIX RDD workareas
+
+  * contrib/hbfoxpro/relfunc.c
+    ! fix InList() FoxPro compatible function not looking at the last
+      parameter passed. Thanks to Attila Szabo for the information
+      posted on the developers list.
+
 2019-01-25 12:26 UTC+0100 Przemyslaw Czerpak (druzus/at/poczta.onet.pl)
   * contrib/xhb/hbxml.c
     ! fixed memory leak

contrib/hbfoxpro/relfunc.c

@@ -72,7 +72,7 @@ HB_FUNC( INLIST )
       PHB_ITEM pValue = hb_param( 1, HB_IT_ANY );
       int iParam;
 
-      for( iParam = 2; iParam < iPCount; ++iParam )
+      for( iParam = 2; iParam <= iPCount; ++iParam )
       {
          if( hb_itemEqual( pValue, hb_param( iParam, HB_IT_ANY ) ) )
          {

contrib/rddsql/sqlbase.c

@@ -539,6 +539,7 @@ static HB_ERRCODE sqlbasePutValue( SQLBASEAREAP pArea, HB_USHORT uiIndex, PHB_IT
        ( HB_IS_NUMBER( pItem ) && ( pField->uiType == HB_FT_INTEGER || pField->uiType == HB_FT_LONG ||
                                     pField->uiType == HB_FT_FLOAT || pField->uiType == HB_FT_DOUBLE ) ) ||
        ( HB_IS_LOGICAL( pItem ) && pField->uiType == HB_FT_LOGICAL ) ||
+       pField->uiType == HB_FT_ANY ||
        HB_IS_NIL( pItem ) )
    {
       hb_arraySet( ( PHB_ITEM ) pArea->pRecord, uiIndex, pItem );
@@ -599,6 +600,31 @@ static HB_ERRCODE sqlbaseRecId( SQLBASEAREAP pArea, PHB_ITEM pRecNo )
 }
 
 
+static HB_ERRCODE sqlbaseZap( SQLBASEAREAP pArea )
+{
+   HB_ULONG ulIndex;
+
+   for( ulIndex = 1; ulIndex <= pArea->ulRecCount; ulIndex++ )
+   {
+      if( pArea->pRowFlags[ ulIndex ] & SQLDD_FLAG_CACHED )
+         hb_itemRelease( ( PHB_ITEM ) pArea->pRow[ ulIndex ] );
+   }
+
+   pArea->ulRecCount = 0;
+   pArea->ulRecNo = 0;
+
+   pArea->pRow = ( void ** ) hb_xrealloc( pArea->pRow, SQLDD_ROWSET_RESIZE * sizeof( void * ) );
+   pArea->pRowFlags = ( HB_BYTE * ) hb_xrealloc( pArea->pRowFlags, SQLDD_ROWSET_RESIZE * sizeof( HB_BYTE ) );
+   pArea->ulRecMax = SQLDD_ROWSET_RESIZE;
+
+   pArea->fFetched = HB_TRUE;
+
+   pArea->fPositioned = HB_FALSE;
+
+   return SELF_GOTOP( &pArea->area );
+}
+
+
 static HB_ERRCODE sqlbaseClose( SQLBASEAREAP pArea )
 {
    if( SELF_GOCOLD( &pArea->area ) == HB_FAILURE )
@@ -1140,7 +1166,7 @@ static RDDFUNCS sqlbaseTable =
    ( DBENTRYP_VS ) NULL,             /* sqlbaseSort */
    ( DBENTRYP_VT ) NULL,             /* sqlbaseTrans */
    ( DBENTRYP_VT ) NULL,             /* sqlbaseTransRec */
-   ( DBENTRYP_V ) NULL,              /* sqlbaseZap */
+   ( DBENTRYP_V ) sqlbaseZap,
    ( DBENTRYP_VR ) NULL,             /* sqlbaseChildEnd */
    ( DBENTRYP_VR ) NULL,             /* sqlbaseChildStart */
    ( DBENTRYP_VR ) NULL,             /* sqlbaseChildSync */

contrib/rddsql/sqlmix.c

@@ -1468,6 +1468,29 @@ static HB_ERRCODE sqlmixGoHot( SQLMIXAREAP pArea )
    return HB_SUCCESS;
 }
 
+static HB_ERRCODE sqlmixZap( SQLMIXAREAP pArea )
+{
+   PMIXTAG pTag;
+
+   if( SUPER_ZAP( &pArea->sqlarea.area ) == HB_FAILURE )
+      return HB_FAILURE;
+
+   pTag = pArea->pTagList;
+
+   while( pTag )
+   {
+      if( pTag->Root )
+         hb_mixTagDestroyNode( pTag->Root );
+
+      pTag->Root = hb_mixTagCreateNode( pTag, HB_TRUE );
+      pTag->fEof = HB_TRUE;
+
+      pTag = pTag->pNext;
+   }
+
+   return HB_SUCCESS;
+}
+
 
 static HB_ERRCODE sqlmixClose( SQLMIXAREAP pArea )
 {
@@ -2013,7 +2036,7 @@ static RDDFUNCS sqlmixTable =
   ( DBENTRYP_VS ) NULL,                 /* sqlmixSort */
   ( DBENTRYP_VT ) NULL,                 /* sqlmixTrans */
   ( DBENTRYP_VT ) NULL,                 /* sqlmixTransRec */
-  ( DBENTRYP_V ) NULL,                  /* sqlmixZap */
+  ( DBENTRYP_V ) sqlmixZap,
   ( DBENTRYP_VR ) NULL,                 /* sqlmixChildEnd */
   ( DBENTRYP_VR ) NULL,                 /* sqlmixChildStart */
   ( DBENTRYP_VR ) NULL,                 /* sqlmixChildSync */

contrib/sddsqlt3/core.c

@@ -48,6 +48,7 @@
 
 #include "hbapiitm.h"
 #include "hbapistr.h"
+#include "hbapicdp.h"
 #include "hbdate.h"
 #include "hbset.h"
 #include "hbvm.h"
@@ -204,7 +205,19 @@ static HB_USHORT sqlite3DeclType(sqlite3_stmt * st, HB_USHORT uiIndex )
       if( hb_strAtI( "REAL", 4, szDeclType, nLen ) != 0 ||
           hb_strAtI( "FLOA", 4, szDeclType, nLen ) != 0 ||
           hb_strAtI( "DOUB", 4, szDeclType, nLen ) != 0 )
+         return HB_FT_LONG; /* logically HB_FT_DOUBLE, what was the idea? */
+#ifdef HB_SQLT3_MAP_DECLARED_EMULATED
+      /* types not handled in a specific way by SQLITE3
+       * but anyway we should try to look at declarations
+       */
+      if( hb_strAtI( "TIME", 4, szDeclType, nLen ) != 0 )
+         return HB_FT_TIMESTAMP;
+      if( hb_strAtI( "DATE", 4, szDeclType, nLen ) != 0 )
+         return HB_FT_DATE;
+      if( hb_strAtI( "NUME", 4, szDeclType, nLen ) != 0 ||
+          hb_strAtI( "NUMB", 4, szDeclType, nLen ) != 0 )
          return HB_FT_LONG;
+#endif
    }
 
 #ifdef HB_SQLT3_MAP_UNDECLARED_TYPES_AS_ANY
@@ -232,6 +245,58 @@ static HB_USHORT sqlite3DeclType(sqlite3_stmt * st, HB_USHORT uiIndex )
 #endif
 }
 
+#ifdef HB_SQLT3_MAP_DECLARED_EMULATED
+static void sqlite3DeclStru( sqlite3_stmt * st, HB_USHORT uiIndex, HB_USHORT * puiLen, HB_USHORT * puiDec )
+{
+   const char * szDeclType = sqlite3_column_decltype( st, uiIndex );
+
+   if( szDeclType != NULL )
+   {
+      HB_SIZE nLen = strlen( szDeclType );
+      HB_SIZE nAt;
+      int iOverflow;
+      HB_MAXINT iRetLen = 0;
+
+      /* SQLite doesn't normally have field size limits,
+       * but column declarations are freeform - let's
+       * try some really stupid guesswork on schema...
+       */
+
+      if( ( nAt = hb_strAt( "(", 1, szDeclType, nLen ) ) > 0 )
+      {
+         if( puiLen )
+         {
+            iRetLen = hb_strValInt( szDeclType + nAt, &iOverflow );
+            if( ! puiDec || ( iRetLen > 0 && iRetLen < 100 ) )
+               * puiLen = ( HB_USHORT ) iRetLen;
+         }
+
+         if( ! puiDec )
+            return;
+
+         if( * puiLen < 2 )
+            * puiDec = 0;
+         else if( puiLen &&
+                  ( nAt = hb_strAt( ",", 1, szDeclType + nAt, nLen - nAt - 1 ) ) > 0 )
+         {
+            if( ( iRetLen = hb_strValInt( szDeclType + nAt, &iOverflow ) ) > 0 )
+            {
+               * puiDec = ( HB_USHORT ) HB_MIN( * puiLen - 1, iRetLen );
+
+               /* SQL column declaration doesn't include space for
+                * decimal separator, while xBase stores it.
+                */
+
+               * puiLen = ( HB_USHORT ) ++iRetLen;
+            }
+            else if( iRetLen == 0 )
+               * puiDec = 0;
+         }
+      }
+   }
+}
+#endif
+
 /* --- SDD METHODS --- */
 static HB_ERRCODE sqlite3Connect( SQLDDCONNECTION * pConnection, PHB_ITEM pItem )
 {
@@ -255,8 +320,10 @@ static HB_ERRCODE sqlite3Disconnect( SQLDDCONNECTION * pConnection )
 {
    HB_ERRCODE errCode;
 
-   errCode = sqlite3_close( ( ( SDDCONN * ) pConnection->pSDDConn )->pDb ) ? HB_SUCCESS : HB_FAILURE;
+   errCode = sqlite3_close( ( ( SDDCONN * ) pConnection->pSDDConn )->pDb ) == SQLITE_OK ? HB_SUCCESS : HB_FAILURE;
-   hb_xfree( pConnection->pSDDConn );
+   if( errCode == HB_SUCCESS )
+      hb_xfree( pConnection->pSDDConn );
+
    return errCode;
 }
 
@@ -300,6 +367,7 @@ static HB_ERRCODE sqlite3Open( SQLBASEAREAP pArea )
    HB_ERRCODE     errCode;
    char *         szError;
    HB_BOOL        bError;
+   int            iStatus, result;
 
    pArea->pSDDData = memset( hb_xgrab( sizeof( SDDDATA ) ), 0, sizeof( SDDDATA ) );
    pSDDData        = ( SDDDATA * ) pArea->pSDDData;
@@ -307,7 +375,13 @@ static HB_ERRCODE sqlite3Open( SQLBASEAREAP pArea )
    pItem    = hb_itemPutC( NULL, pArea->szQuery );
    pszQuery = S_HB_ITEMGETSTR( pItem, &hQuery, &nQueryLen );
 
-   if( sqlite3_prepare_v2( pDb, pszQuery, ( int ) nQueryLen, &st, NULL ) != SQLITE_OK )
+#if SQLITE_VERSION_NUMBER >= 3020000
+   result = sqlite3_prepare_v3( pDb, pszQuery, ( int ) nQueryLen, 0, &st, NULL );
+#else
+   result = sqlite3_prepare_v2( pDb, pszQuery, ( int ) nQueryLen, &st, NULL );
+#endif
+
+   if( result != SQLITE_OK )
    {
       hb_strfree( hQuery );
       hb_itemRelease( pItem );
@@ -323,7 +397,9 @@ static HB_ERRCODE sqlite3Open( SQLBASEAREAP pArea )
       hb_itemRelease( pItem );
    }
 
-   if( sqlite3_step( st ) != SQLITE_ROW )
+   if( ( iStatus = sqlite3_step( st ) ) == SQLITE_DONE )
+      pArea->fFetched = HB_TRUE;
+   else if( iStatus != SQLITE_ROW )
    {
       szError = sqlite3GetError( pDb, &errCode );
       hb_errRT_SQLT3DD( EG_OPEN, ESQLDD_INVALIDQUERY, szError, pArea->szQuery, errCode );
@@ -341,10 +417,20 @@ static HB_ERRCODE sqlite3Open( SQLBASEAREAP pArea )
    for( uiIndex = 0; uiIndex < uiFields; ++uiIndex )
    {
       DBFIELDINFO dbFieldInfo;
-
+#ifdef HB_SQLT3_FIELDNAME_STRICT
+      HB_SIZE nPos;
+#endif
       memset( &dbFieldInfo, 0, sizeof( dbFieldInfo ) );
       pName = S_HB_ITEMPUTSTR( pName, sqlite3_column_name( st, uiIndex ) );
-      dbFieldInfo.atomName = hb_itemGetCPtr( pName );
+
+#ifdef HB_SQLT3_FIELDNAME_STRICT
+      /* WA->T.FIELD syntax is not valid, but FieldPos("t.field") is OK */
+      if( ( nPos = hb_strAt( ".", 1, hb_itemGetCPtr( pName ), hb_itemGetCLen( pName ) ) ) != 0 )
+         dbFieldInfo.atomName = hb_itemGetCPtr( pName ) + nPos;
+      else
+#endif
+         dbFieldInfo.atomName = hb_itemGetCPtr( pName );
+
       dbFieldInfo.uiType = sqlite3DeclType( st, uiIndex );
       pItem = hb_arrayGetItemPtr( pItemEof, uiIndex + 1 );
 
@@ -358,10 +444,21 @@ static HB_ERRCODE sqlite3Open( SQLBASEAREAP pArea )
       {
          case HB_FT_STRING:
          {
-            int iSize = sqlite3_column_bytes( st, uiIndex );
+            HB_SIZE nSize = hb_cdpUTF8StringLength( ( const char * ) sqlite3_column_text( st, uiIndex ),
-            char * pStr;
+                                                    sqlite3_column_bytes( st, uiIndex ) );
 
-            dbFieldInfo.uiLen = ( HB_USHORT ) HB_MAX( iSize, 10 );
+            /* sqlite3_column_bytes() returns variable lengths for UTF-8
+               strings - *_bytes16() UTF-16 could do that too, but not
+               for mostly used character sets. Yet seems better to use
+               hb_cdpUTF8StringLength() */
+
+            char * pStr;
+            HB_USHORT uiRetLen = 10;
+
+#ifdef HB_SQLT3_MAP_DECLARED_EMULATED
+            sqlite3DeclStru( st, uiIndex, &uiRetLen, NULL );
+#endif
+            dbFieldInfo.uiLen = ( HB_USHORT ) HB_MAX( nSize, uiRetLen );
             pStr = ( char * ) hb_xgrab( ( HB_SIZE ) dbFieldInfo.uiLen + 1 );
             memset( pStr, ' ', dbFieldInfo.uiLen );
             hb_itemPutCLPtr( pItem, pStr, dbFieldInfo.uiLen );
@@ -380,9 +477,24 @@ static HB_ERRCODE sqlite3Open( SQLBASEAREAP pArea )
          case HB_FT_LONG:
             dbFieldInfo.uiLen = 20;
             dbFieldInfo.uiDec = ( HB_USHORT ) hb_setGetDecimals();
+#ifdef HB_SQLT3_MAP_DECLARED_EMULATED
+            sqlite3DeclStru( st, uiIndex, &dbFieldInfo.uiLen, &dbFieldInfo.uiDec );
+#endif
             hb_itemPutNDDec( pItem, 0.0, dbFieldInfo.uiDec );
             break;
 
+#ifdef HB_SQLT3_MAP_DECLARED_EMULATED
+         case HB_FT_DATE:
+            dbFieldInfo.uiLen = 8;
+            hb_itemPutDS( pItem, NULL );
+            break;
+
+         case HB_FT_TIMESTAMP:
+            dbFieldInfo.uiLen = 8;
+            hb_itemPutTDT( pItem, 0, 0 );
+            break;
+#endif
+
          case HB_FT_ANY:
             dbFieldInfo.uiLen = 6;
             break;
@@ -486,6 +598,40 @@ static HB_ERRCODE sqlite3GoTo( SQLBASEAREAP pArea, HB_ULONG ulRecNo )
                pItem = hb_itemPutNDDec( NULL, sqlite3_column_double( st, ui ), pField->uiDec );
                break;
 
+#ifdef HB_SQLT3_MAP_DECLARED_EMULATED
+            case HB_FT_DATE:
+               if( sqlite3_column_bytes( st, ui ) >= 10 )
+               {
+                  char szDate[ 9 ];
+                  const char * pValue = ( const char * ) sqlite3_column_text( st, ui );
+
+                  szDate[ 0 ] = pValue[ 0 ];
+                  szDate[ 1 ] = pValue[ 1 ];
+                  szDate[ 2 ] = pValue[ 2 ];
+                  szDate[ 3 ] = pValue[ 3 ];
+                  szDate[ 4 ] = pValue[ 5 ];
+                  szDate[ 5 ] = pValue[ 6 ];
+                  szDate[ 6 ] = pValue[ 8 ];
+                  szDate[ 7 ] = pValue[ 9 ];
+                  szDate[ 8 ] = '\0';
+                  pItem = hb_itemPutDS( NULL, szDate );
+               }
+               else if( sqlite3_column_bytes( st, ui ) == 8 )
+                  pItem = hb_itemPutDS( NULL, ( const char * ) sqlite3_column_text( st, ui ) );
+
+               break;
+
+            case HB_FT_TIMESTAMP:
+               if( sqlite3_column_bytes( st, ui ) >= 10 )
+               {
+                  long lDate, lTime;
+                  const char * pValue = ( const char * ) sqlite3_column_text( st, ui );
+
+                  hb_timeStampStrGetDT( pValue, &lDate, &lTime );
+                  pItem = hb_itemPutTDT( NULL, lDate, lTime );
+                  break;
+               }
+#endif
             case HB_FT_BLOB:
                pItem = hb_itemPutCL( NULL, ( const char * ) sqlite3_column_blob( st, ui ), sqlite3_column_bytes( st, ui ) );
                break;