diff --git a/harbour/ChangeLog b/harbour/ChangeLog index 50566382bf..22e035daa2 100644 --- a/harbour/ChangeLog +++ b/harbour/ChangeLog @@ -16,6 +16,45 @@ The license applies to all entries newer than 2009-04-28. */ +2011-07-30 00:00 UTC+0200 Viktor Szakats (harbour.01 syenar.hu) + * src/3rd/png/pngerror.c + * src/3rd/png/pngwrite.c + * src/3rd/png/pngconf.h + * src/3rd/png/pngwtran.c + * src/3rd/png/pngset.c + * src/3rd/png/pngpread.c + * src/3rd/png/LICENSE + * src/3rd/png/pngrutil.c + * src/3rd/png/png.dif + * src/3rd/png/pnginfo.h + * src/3rd/png/pngstru.h + * src/3rd/png/pngwutil.c + * src/3rd/png/pnglconf.h + * src/3rd/png/pngread.c + * src/3rd/png/pngmem.c + * src/3rd/png/png.c + * src/3rd/png/png.h + * src/3rd/png/pngpriv.h + * src/3rd/png/pngrtran.c + * src/3rd/png/Makefile + * src/3rd/png/pngtrans.c + * update to 1.5.4 + + * contrib/3rd/sqlite3/sqlite3.c + * contrib/3rd/sqlite3/sqlite3.dif + * contrib/3rd/sqlite3/sqlite3.h + * contrib/3rd/sqlite3/sqlite3.hbp + * update to 3.7.7.1 + + ; Both patch from Tamas Tevesz. Thank you. + + ; Administrative note: I'm also giving up applying patches, + so I'm looking for devs taking over this task. I'd be + most happy if Tamas could commit directly his own, since + he is doing a great job and the patches almost never require + any touchups, so I'm gladly opening R/W access for direct + commit. Tamas, what do you think? + 2011-07-29 22:50 UTC+0200 Viktor Szakats (harbour.01 syenar.hu) * package/winuni/RELNOTES ! adjusted hbmk2 cmdline example for building wce apps diff --git a/harbour/contrib/3rd/sqlite3/sqlite3.c b/harbour/contrib/3rd/sqlite3/sqlite3.c index 5ee945c9b6..b3185c9c2b 100644 --- a/harbour/contrib/3rd/sqlite3/sqlite3.c +++ b/harbour/contrib/3rd/sqlite3/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.7.6.3. By combining all the individual C code files into this +** version 3.7.7.1. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -650,9 +650,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.6.3" -#define SQLITE_VERSION_NUMBER 3007006 -#define SQLITE_SOURCE_ID "2011-05-19 13:26:54 ed1da510a239ea767a01dc332b667119fa3c908e" +#define SQLITE_VERSION "3.7.7.1" +#define SQLITE_VERSION_NUMBER 3007007 +#define SQLITE_SOURCE_ID "2011-06-28 17:39:05 af0d91adf497f5f36ec3813f04235a6e195a605f" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -853,7 +853,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** argument. ^If the callback function of the 3rd argument to ** sqlite3_exec() is not NULL, then it is invoked for each result row ** coming out of the evaluated SQL statements. ^The 4th argument to -** to sqlite3_exec() is relayed through to the 1st argument of each +** sqlite3_exec() is relayed through to the 1st argument of each ** callback invocation. ^If the callback pointer to sqlite3_exec() ** is NULL, then no callback is ever invoked and result rows are ** ignored. @@ -918,7 +918,8 @@ SQLITE_API int sqlite3_exec( ** ** New error codes may be added in future versions of SQLite. ** -** See also: [SQLITE_IOERR_READ | extended result codes] +** See also: [SQLITE_IOERR_READ | extended result codes], +** [sqlite3_vtab_on_conflict()] [SQLITE_ROLLBACK | result codes]. */ #define SQLITE_OK 0 /* Successful result */ /* beginning-of-error-codes */ @@ -995,17 +996,21 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8)) #define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8)) #define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) +#define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) +#define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) +#define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) +#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations ** ** These bit values are intended for use in the ** 3rd parameter to the [sqlite3_open_v2()] interface and -** in the 4th parameter to the xOpen method of the -** [sqlite3_vfs] object. +** in the 4th parameter to the [sqlite3_vfs.xOpen] method. */ #define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_READWRITE 0x00000002 /* Ok for sqlite3_open_v2() */ @@ -1013,6 +1018,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_OPEN_DELETEONCLOSE 0x00000008 /* VFS only */ #define SQLITE_OPEN_EXCLUSIVE 0x00000010 /* VFS only */ #define SQLITE_OPEN_AUTOPROXY 0x00000020 /* VFS only */ +#define SQLITE_OPEN_URI 0x00000040 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_MAIN_DB 0x00000100 /* VFS only */ #define SQLITE_OPEN_TEMP_DB 0x00000200 /* VFS only */ #define SQLITE_OPEN_TRANSIENT_DB 0x00000400 /* VFS only */ @@ -1123,17 +1129,18 @@ struct sqlite3_file { /* ** CAPI3REF: OS Interface File Virtual Methods Object ** -** Every file opened by the [sqlite3_vfs] xOpen method populates an +** Every file opened by the [sqlite3_vfs.xOpen] method populates an ** [sqlite3_file] object (or, more commonly, a subclass of the ** [sqlite3_file] object) with a pointer to an instance of this object. ** This object defines the methods used to perform various operations ** against the open file represented by the [sqlite3_file] object. ** -** If the xOpen method sets the sqlite3_file.pMethods element +** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element ** to a non-NULL pointer, then the sqlite3_io_methods.xClose method -** may be invoked even if the xOpen reported that it failed. The -** only way to prevent a call to xClose following a failed xOpen -** is for the xOpen to set the sqlite3_file.pMethods element to NULL. +** may be invoked even if the [sqlite3_vfs.xOpen] reported that it failed. The +** only way to prevent a call to xClose following a failed [sqlite3_vfs.xOpen] +** is for the [sqlite3_vfs.xOpen] to set the sqlite3_file.pMethods element +** to NULL. ** ** The flags argument to xSync may be one of [SQLITE_SYNC_NORMAL] or ** [SQLITE_SYNC_FULL]. The first choice is the normal fsync(). @@ -1305,7 +1312,8 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** ** An instance of the sqlite3_vfs object defines the interface between ** the SQLite core and the underlying operating system. The "vfs" -** in the name of the object stands for "virtual file system". +** in the name of the object stands for "virtual file system". See +** the [VFS | VFS documentation] for further information. ** ** The value of the iVersion field is initially 1 but may be larger in ** future versions of SQLite. Additional fields may be appended to this @@ -1334,6 +1342,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** The zName field holds the name of the VFS module. The name must ** be unique across all VFS modules. ** +** [[sqlite3_vfs.xOpen]] ** ^SQLite guarantees that the zFilename parameter to xOpen ** is either a NULL pointer or string obtained ** from xFullPathname() with an optional suffix added. @@ -1411,6 +1420,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** element will be valid after xOpen returns regardless of the success ** or failure of the xOpen call. ** +** [[sqlite3_vfs.xAccess]] ** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] @@ -1435,7 +1445,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** method returns a Julian Day Number for the current date and time as ** a floating point value. ** ^The xCurrentTimeInt64() method returns, as an integer, the Julian -** Day Number multipled by 86400000 (the number of milliseconds in +** Day Number multiplied by 86400000 (the number of milliseconds in ** a 24-hour day). ** ^SQLite will use the xCurrentTimeInt64() method to get the current ** date and time if that method is available (if iVersion is 2 or @@ -1657,9 +1667,9 @@ SQLITE_API int sqlite3_os_end(void); ** implementation of an application-defined [sqlite3_os_init()]. ** ** The first argument to sqlite3_config() is an integer -** [SQLITE_CONFIG_SINGLETHREAD | configuration option] that determines +** [configuration option] that determines ** what property of SQLite is to be configured. Subsequent arguments -** vary depending on the [SQLITE_CONFIG_SINGLETHREAD | configuration option] +** vary depending on the [configuration option] ** in the first argument. ** ** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. @@ -1769,6 +1779,7 @@ struct sqlite3_mem_methods { /* ** CAPI3REF: Configuration Options +** KEYWORDS: {configuration option} ** ** These constants are the available integer configuration options that ** can be passed as the first argument to the [sqlite3_config()] interface. @@ -1781,7 +1792,7 @@ struct sqlite3_mem_methods { ** is invoked. ** **
-**
SQLITE_CONFIG_SINGLETHREAD
+** [[SQLITE_CONFIG_SINGLETHREAD]]
SQLITE_CONFIG_SINGLETHREAD
**
There are no arguments to this option. ^This option sets the ** [threading mode] to Single-thread. In other words, it disables ** all mutexing and puts SQLite into a mode where it can only be used @@ -1792,7 +1803,7 @@ struct sqlite3_mem_methods { ** [SQLITE_ERROR] if called with the SQLITE_CONFIG_SINGLETHREAD ** configuration option.
** -**
SQLITE_CONFIG_MULTITHREAD
+** [[SQLITE_CONFIG_MULTITHREAD]]
SQLITE_CONFIG_MULTITHREAD
**
There are no arguments to this option. ^This option sets the ** [threading mode] to Multi-thread. In other words, it disables ** mutexing on [database connection] and [prepared statement] objects. @@ -1806,7 +1817,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] will return [SQLITE_ERROR] if called with the ** SQLITE_CONFIG_MULTITHREAD configuration option.
** -**
SQLITE_CONFIG_SERIALIZED
+** [[SQLITE_CONFIG_SERIALIZED]]
SQLITE_CONFIG_SERIALIZED
**
There are no arguments to this option. ^This option sets the ** [threading mode] to Serialized. In other words, this option enables ** all mutexes including the recursive @@ -1822,7 +1833,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] will return [SQLITE_ERROR] if called with the ** SQLITE_CONFIG_SERIALIZED configuration option.
** -**
SQLITE_CONFIG_MALLOC
+** [[SQLITE_CONFIG_MALLOC]]
SQLITE_CONFIG_MALLOC
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mem_methods] structure. The argument specifies ** alternative low-level memory allocation routines to be used in place of @@ -1830,7 +1841,7 @@ struct sqlite3_mem_methods { ** its own private copy of the content of the [sqlite3_mem_methods] structure ** before the [sqlite3_config()] call returns.
** -**
SQLITE_CONFIG_GETMALLOC
+** [[SQLITE_CONFIG_GETMALLOC]]
SQLITE_CONFIG_GETMALLOC
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ @@ -1838,7 +1849,7 @@ struct sqlite3_mem_methods { ** routines with a wrapper that simulations memory allocation failure or ** tracks memory usage, for example.
** -**
SQLITE_CONFIG_MEMSTATUS
+** [[SQLITE_CONFIG_MEMSTATUS]]
SQLITE_CONFIG_MEMSTATUS
**
^This option takes single argument of type int, interpreted as a ** boolean, which enables or disables the collection of memory allocation ** statistics. ^(When memory allocation statistics are disabled, the @@ -1854,7 +1865,7 @@ struct sqlite3_mem_methods { ** allocation statistics are disabled by default. **
** -**
SQLITE_CONFIG_SCRATCH
+** [[SQLITE_CONFIG_SCRATCH]]
SQLITE_CONFIG_SCRATCH
**
^This option specifies a static memory buffer that SQLite can use for ** scratch memory. There are three arguments: A pointer an 8-byte ** aligned memory buffer from which the scratch allocations will be @@ -1870,9 +1881,9 @@ struct sqlite3_mem_methods { ** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed.
** -**
SQLITE_CONFIG_PAGECACHE
+** [[SQLITE_CONFIG_PAGECACHE]]
SQLITE_CONFIG_PAGECACHE
**
^This option specifies a static memory buffer that SQLite can use for -** the database page cache with the default page cache implemenation. +** the database page cache with the default page cache implementation. ** This configuration should not be used if an application-define page ** cache implementation is loaded using the SQLITE_CONFIG_PCACHE option. ** There are three arguments to this option: A pointer to 8-byte aligned @@ -1891,7 +1902,7 @@ struct sqlite3_mem_methods { ** be aligned to an 8-byte boundary or subsequent behavior of SQLite ** will be undefined.
** -**
SQLITE_CONFIG_HEAP
+** [[SQLITE_CONFIG_HEAP]]
SQLITE_CONFIG_HEAP
**
^This option specifies a static memory buffer that SQLite will use ** for all of its dynamic memory allocation needs beyond those provided ** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. @@ -1908,7 +1919,7 @@ struct sqlite3_mem_methods { ** The minimum allocation size is capped at 2^12. Reasonable values ** for the minimum allocation size are 2^5 through 2^8.
** -**
SQLITE_CONFIG_MUTEX
+** [[SQLITE_CONFIG_MUTEX]]
SQLITE_CONFIG_MUTEX
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mutex_methods] structure. The argument specifies ** alternative low-level mutex routines to be used in place @@ -1920,7 +1931,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] with the SQLITE_CONFIG_MUTEX configuration option will ** return [SQLITE_ERROR].
** -**
SQLITE_CONFIG_GETMUTEX
+** [[SQLITE_CONFIG_GETMUTEX]]
SQLITE_CONFIG_GETMUTEX
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mutex_methods] structure. The ** [sqlite3_mutex_methods] @@ -1933,7 +1944,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] with the SQLITE_CONFIG_GETMUTEX configuration option will ** return [SQLITE_ERROR].
** -**
SQLITE_CONFIG_LOOKASIDE
+** [[SQLITE_CONFIG_LOOKASIDE]]
SQLITE_CONFIG_LOOKASIDE
**
^(This option takes two arguments that determine the default ** memory allocation for the lookaside memory allocator on each ** [database connection]. The first argument is the @@ -1943,18 +1954,18 @@ struct sqlite3_mem_methods { ** verb to [sqlite3_db_config()] can be used to change the lookaside ** configuration on individual connections.)^
** -**
SQLITE_CONFIG_PCACHE
+** [[SQLITE_CONFIG_PCACHE]]
SQLITE_CONFIG_PCACHE
**
^(This option takes a single argument which is a pointer to ** an [sqlite3_pcache_methods] object. This object specifies the interface ** to a custom page cache implementation.)^ ^SQLite makes a copy of the ** object and uses it for page cache memory allocations.
** -**
SQLITE_CONFIG_GETPCACHE
+** [[SQLITE_CONFIG_GETPCACHE]]
SQLITE_CONFIG_GETPCACHE
**
^(This option takes a single argument which is a pointer to an ** [sqlite3_pcache_methods] object. SQLite copies of the current ** page cache implementation into that object.)^
** -**
SQLITE_CONFIG_LOG
+** [[SQLITE_CONFIG_LOG]]
SQLITE_CONFIG_LOG
**
^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is @@ -1972,6 +1983,18 @@ struct sqlite3_mem_methods { ** In a multi-threaded application, the application-defined logger ** function must be threadsafe.
** +** [[SQLITE_CONFIG_URI]]
SQLITE_CONFIG_URI +**
This option takes a single argument of type int. If non-zero, then +** URI handling is globally enabled. If the parameter is zero, then URI handling +** is globally disabled. If URI handling is globally enabled, all filenames +** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +** specified as part of [ATTACH] commands are interpreted as URIs, regardless +** of whether or not the [SQLITE_OPEN_URI] flag is set when the database +** connection is opened. If it is globally disabled, filenames are +** only interpreted as URIs if the SQLITE_OPEN_URI flag is set when the +** database connection is opened. By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** [SQLITE_USE_URI] symbol defined. **
*/ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1990,6 +2013,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_PCACHE 14 /* sqlite3_pcache_methods* */ #define SQLITE_CONFIG_GETPCACHE 15 /* sqlite3_pcache_methods* */ #define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ /* ** CAPI3REF: Database Connection Configuration Options @@ -2075,13 +2099,17 @@ SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); ** ** ^This routine returns the [rowid] of the most recent ** successful [INSERT] into the database from the [database connection] -** in the first argument. ^If no successful [INSERT]s +** in the first argument. ^As of SQLite version 3.7.7, this routines +** records the last insert rowid of both ordinary tables and [virtual tables]. +** ^If no successful [INSERT]s ** have ever occurred on that database connection, zero is returned. ** -** ^(If an [INSERT] occurs within a trigger, then the [rowid] of the inserted -** row is returned by this routine as long as the trigger is running. -** But once the trigger terminates, the value returned by this routine -** reverts to the last value inserted before the trigger fired.)^ +** ^(If an [INSERT] occurs within a trigger or within a [virtual table] +** method, then this routine will return the [rowid] of the inserted +** row as long as the trigger or virtual table method is running. +** But once the trigger or virtual table method ends, the value returned +** by this routine reverts to what it was before the trigger or virtual +** table method began.)^ ** ** ^An [INSERT] that fails due to a constraint violation is not a ** successful [INSERT] and does not change the value returned by this @@ -2744,6 +2772,9 @@ SQLITE_API int sqlite3_set_authorizer( ** to signal SQLite whether or not the action is permitted. See the ** [sqlite3_set_authorizer | authorizer documentation] for additional ** information. +** +** Note that SQLITE_IGNORE is also used as a [SQLITE_ROLLBACK | return code] +** from the [sqlite3_vtab_on_conflict()] interface. */ #define SQLITE_DENY 1 /* Abort the SQL statement with an error */ #define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */ @@ -2866,7 +2897,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); /* ** CAPI3REF: Opening A New Database Connection ** -** ^These routines open an SQLite database file whose name is given by the +** ^These routines open an SQLite database file as specified by the ** filename argument. ^The filename argument is interpreted as UTF-8 for ** sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte ** order for sqlite3_open16(). ^(A [database connection] handle is usually @@ -2893,7 +2924,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** sqlite3_open_v2() can take one of ** the following three values, optionally combined with the ** [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], [SQLITE_OPEN_SHAREDCACHE], -** and/or [SQLITE_OPEN_PRIVATECACHE] flags:)^ +** [SQLITE_OPEN_PRIVATECACHE], and/or [SQLITE_OPEN_URI] flags:)^ ** **
** ^(
[SQLITE_OPEN_READONLY]
@@ -2912,9 +2943,8 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); **
** ** If the 3rd parameter to sqlite3_open_v2() is not one of the -** combinations shown above or one of the combinations shown above combined -** with the [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], -** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_PRIVATECACHE] flags, +** combinations shown above optionally combined with other +** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits] ** then the behavior is undefined. ** ** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection @@ -2929,6 +2959,11 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** [SQLITE_OPEN_PRIVATECACHE] flag causes the database connection to not ** participate in [shared cache mode] even if it is enabled. ** +** ^The fourth parameter to sqlite3_open_v2() is the name of the +** [sqlite3_vfs] object that defines the operating system interface that +** the new database connection should use. ^If the fourth parameter is +** a NULL pointer then the default [sqlite3_vfs] object is used. +** ** ^If the filename is ":memory:", then a private, temporary in-memory database ** is created for the connection. ^This in-memory database will vanish when ** the database connection is closed. Future versions of SQLite might @@ -2941,10 +2976,111 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** on-disk database will be created. ^This private database will be ** automatically deleted as soon as the database connection is closed. ** -** ^The fourth parameter to sqlite3_open_v2() is the name of the -** [sqlite3_vfs] object that defines the operating system interface that -** the new database connection should use. ^If the fourth parameter is -** a NULL pointer then the default [sqlite3_vfs] object is used. +** [[URI filenames in sqlite3_open()]]

URI Filenames

+** +** ^If [URI filename] interpretation is enabled, and the filename argument +** begins with "file:", then the filename is interpreted as a URI. ^URI +** filename interpretation is enabled if the [SQLITE_OPEN_URI] flag is +** set in the fourth argument to sqlite3_open_v2(), or if it has +** been enabled globally using the [SQLITE_CONFIG_URI] option with the +** [sqlite3_config()] method or by the [SQLITE_USE_URI] compile-time option. +** As of SQLite version 3.7.7, URI filename interpretation is turned off +** by default, but future releases of SQLite might enable URI filename +** interpretation by default. See "[URI filenames]" for additional +** information. +** +** URI filenames are parsed according to RFC 3986. ^If the URI contains an +** authority, then it must be either an empty string or the string +** "localhost". ^If the authority is not an empty string or "localhost", an +** error is returned to the caller. ^The fragment component of a URI, if +** present, is ignored. +** +** ^SQLite uses the path component of the URI as the name of the disk file +** which contains the database. ^If the path begins with a '/' character, +** then it is interpreted as an absolute path. ^If the path does not begin +** with a '/' (meaning that the authority section is omitted from the URI) +** then the path is interpreted as a relative path. +** ^On windows, the first component of an absolute path +** is a drive specification (e.g. "C:"). +** +** [[core URI query parameters]] +** The query component of a URI may contain parameters that are interpreted +** either by SQLite itself, or by a [VFS | custom VFS implementation]. +** SQLite interprets the following three query parameters: +** +** +** +** ^Specifying an unknown parameter in the query component of a URI is not an +** error. Future versions of SQLite might understand additional query +** parameters. See "[query parameters with special meaning to SQLite]" for +** additional information. +** +** [[URI filename examples]]

URI filename examples

+** +** +**
URI filenames Results +**
file:data.db +** Open the file "data.db" in the current directory. +**
file:/home/fred/data.db
+** file:///home/fred/data.db
+** file://localhost/home/fred/data.db
+** Open the database file "/home/fred/data.db". +**
file://darkstar/home/fred/data.db +** An error. "darkstar" is not a recognized authority. +**
+** file:///C:/Documents%20and%20Settings/fred/Desktop/data.db +** Windows only: Open the file "data.db" on fred's desktop on drive +** C:. Note that the %20 escaping in this example is not strictly +** necessary - space characters can be used literally +** in URI filenames. +**
file:data.db?mode=ro&cache=private +** Open file "data.db" in the current directory for read-only access. +** Regardless of whether or not shared-cache mode is enabled by +** default, use a private cache. +**
file:/home/fred/data.db?vfs=unix-nolock +** Open file "/home/fred/data.db". Use the special VFS "unix-nolock". +**
file:data.db?mode=readonly +** An error. "readonly" is not a valid option for the "mode" parameter. +**
+** +** ^URI hexadecimal escape sequences (%HH) are supported within the path and +** query components of a URI. A hexadecimal escape sequence consists of a +** percent sign - "%" - followed by exactly two hexadecimal digits +** specifying an octet value. ^Before the path or query components of a +** URI filename are interpreted, they are encoded using UTF-8 and all +** hexadecimal escape sequences replaced by a single byte containing the +** corresponding octet. If this process generates an invalid UTF-8 encoding, +** the results are undefined. ** ** Note to Windows users: The encoding used for the filename argument ** of sqlite3_open() and sqlite3_open_v2() must be UTF-8, not whatever @@ -2967,6 +3103,26 @@ SQLITE_API int sqlite3_open_v2( const char *zVfs /* Name of VFS module to use */ ); +/* +** CAPI3REF: Obtain Values For URI Parameters +** +** This is a utility routine, useful to VFS implementations, that checks +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of the query parameter. +** +** The zFilename argument is the filename pointer passed into the xOpen() +** method of a VFS implementation. The zParam argument is the name of the +** query parameter we seek. This routine returns the value of the zParam +** parameter if it exists. If the parameter does not exist, this routine +** returns a NULL pointer. +** +** If the zFilename argument to this function is not a pointer that SQLite +** passed into the xOpen VFS method, then the behavior of this routine +** is undefined and probably undesirable. +*/ +SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); + + /* ** CAPI3REF: Error Codes And Messages ** @@ -3082,43 +3238,45 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** Additional information is available at [limits | Limits in SQLite]. ** **
-** ^(
SQLITE_LIMIT_LENGTH
+** [[SQLITE_LIMIT_LENGTH]] ^(
SQLITE_LIMIT_LENGTH
**
The maximum size of any string or BLOB or table row, in bytes.
)^ ** -** ^(
SQLITE_LIMIT_SQL_LENGTH
+** [[SQLITE_LIMIT_SQL_LENGTH]] ^(
SQLITE_LIMIT_SQL_LENGTH
**
The maximum length of an SQL statement, in bytes.
)^ ** -** ^(
SQLITE_LIMIT_COLUMN
+** [[SQLITE_LIMIT_COLUMN]] ^(
SQLITE_LIMIT_COLUMN
**
The maximum number of columns in a table definition or in the ** result set of a [SELECT] or the maximum number of columns in an index ** or in an ORDER BY or GROUP BY clause.
)^ ** -** ^(
SQLITE_LIMIT_EXPR_DEPTH
+** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(
SQLITE_LIMIT_EXPR_DEPTH
**
The maximum depth of the parse tree on any expression.
)^ ** -** ^(
SQLITE_LIMIT_COMPOUND_SELECT
+** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(
SQLITE_LIMIT_COMPOUND_SELECT
**
The maximum number of terms in a compound SELECT statement.
)^ ** -** ^(
SQLITE_LIMIT_VDBE_OP
+** [[SQLITE_LIMIT_VDBE_OP]] ^(
SQLITE_LIMIT_VDBE_OP
**
The maximum number of instructions in a virtual machine program ** used to implement an SQL statement. This limit is not currently ** enforced, though that might be added in some future release of ** SQLite.
)^ ** -** ^(
SQLITE_LIMIT_FUNCTION_ARG
+** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
SQLITE_LIMIT_FUNCTION_ARG
**
The maximum number of arguments on a function.
)^ ** -** ^(
SQLITE_LIMIT_ATTACHED
+** [[SQLITE_LIMIT_ATTACHED]] ^(
SQLITE_LIMIT_ATTACHED
**
The maximum number of [ATTACH | attached databases].)^
** +** [[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]] ** ^(
SQLITE_LIMIT_LIKE_PATTERN_LENGTH
**
The maximum length of the pattern argument to the [LIKE] or ** [GLOB] operators.
)^ ** +** [[SQLITE_LIMIT_VARIABLE_NUMBER]] ** ^(
SQLITE_LIMIT_VARIABLE_NUMBER
**
The maximum index number of any [parameter] in an SQL statement.)^ ** -** ^(
SQLITE_LIMIT_TRIGGER_DEPTH
+** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
SQLITE_LIMIT_TRIGGER_DEPTH
**
The maximum depth of recursion for triggers.
)^ **
*/ @@ -3647,7 +3805,7 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** ^[SQLITE_BUSY] means that the database engine was unable to acquire the ** database locks it needs to do its job. ^If the statement is a [COMMIT] ** or occurs outside of an explicit transaction, then you can retry the -** statement. If the statement is not a [COMMIT] and occurs within a +** statement. If the statement is not a [COMMIT] and occurs within an ** explicit transaction then you should rollback the transaction before ** continuing. ** @@ -3926,7 +4084,7 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); ** CAPI3REF: Destroy A Prepared Statement Object ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. -** ^If the most recent evaluation of the statement encountered no errors or +** ^If the most recent evaluation of the statement encountered no errors ** or if the statement is never been evaluated, then sqlite3_finalize() returns ** SQLITE_OK. ^If the most recent evaluation of statement S failed, then ** sqlite3_finalize(S) returns the appropriate [error code] or @@ -5153,6 +5311,11 @@ struct sqlite3_module { void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), void **ppArg); int (*xRename)(sqlite3_vtab *pVtab, const char *zNew); + /* The methods above are in version 1 of the sqlite_module object. Those + ** below are for version 2 and greater. */ + int (*xSavepoint)(sqlite3_vtab *pVTab, int); + int (*xRelease)(sqlite3_vtab *pVTab, int); + int (*xRollbackTo)(sqlite3_vtab *pVTab, int); }; /* @@ -5835,7 +5998,7 @@ struct sqlite3_mutex_methods { ** ** ^If the argument to sqlite3_mutex_held() is a NULL pointer then ** the routine should return 1. This seems counter-intuitive since -** clearly the mutex cannot be held if it does not exist. But the +** clearly the mutex cannot be held if it does not exist. But ** the reason the mutex does not exist is because the build is not ** using mutexes. And we do not want the assert() containing the ** call to sqlite3_mutex_held() to fail, so a non-zero return is @@ -5958,7 +6121,8 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_PGHDRSZ 17 #define SQLITE_TESTCTRL_SCRATCHMALLOC 18 -#define SQLITE_TESTCTRL_LAST 18 +#define SQLITE_TESTCTRL_LOCALTIME_FAULT 19 +#define SQLITE_TESTCTRL_LAST 19 /* ** CAPI3REF: SQLite Runtime Status @@ -5967,7 +6131,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for ** the specific parameter to measure. ^(Recognized integer codes -** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...].)^ +** are of the form [status parameters | SQLITE_STATUS_...].)^ ** ^The current value of the parameter is returned into *pCurrent. ** ^The highest recorded value is returned in *pHighwater. ^If the ** resetFlag is true, then the highest record value is reset after @@ -5994,12 +6158,13 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF /* ** CAPI3REF: Status Parameters +** KEYWORDS: {status parameters} ** ** These integer constants designate various run-time status parameters ** that can be returned by [sqlite3_status()]. ** **
-** ^(
SQLITE_STATUS_MEMORY_USED
+** [[SQLITE_STATUS_MEMORY_USED]] ^(
SQLITE_STATUS_MEMORY_USED
**
This parameter is the current amount of memory checked out ** using [sqlite3_malloc()], either directly or indirectly. The ** figure includes calls made to [sqlite3_malloc()] by the application @@ -6009,23 +6174,24 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** this parameter. The amount returned is the sum of the allocation ** sizes as reported by the xSize method in [sqlite3_mem_methods].
)^ ** -** ^(
SQLITE_STATUS_MALLOC_SIZE
+** [[SQLITE_STATUS_MALLOC_SIZE]] ^(
SQLITE_STATUS_MALLOC_SIZE
**
This parameter records the largest memory allocation request ** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their ** internal equivalents). Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
)^ ** -** ^(
SQLITE_STATUS_MALLOC_COUNT
+** [[SQLITE_STATUS_MALLOC_COUNT]] ^(
SQLITE_STATUS_MALLOC_COUNT
**
This parameter records the number of separate memory allocations ** currently checked out.
)^ ** -** ^(
SQLITE_STATUS_PAGECACHE_USED
+** [[SQLITE_STATUS_PAGECACHE_USED]] ^(
SQLITE_STATUS_PAGECACHE_USED
**
This parameter returns the number of pages used out of the ** [pagecache memory allocator] that was configured using ** [SQLITE_CONFIG_PAGECACHE]. The ** value returned is in pages, not in bytes.
)^ ** +** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]] ** ^(
SQLITE_STATUS_PAGECACHE_OVERFLOW
**
This parameter returns the number of bytes of page cache ** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] @@ -6035,13 +6201,13 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because ** no space was left in the page cache.
)^ ** -** ^(
SQLITE_STATUS_PAGECACHE_SIZE
+** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(
SQLITE_STATUS_PAGECACHE_SIZE
**
This parameter records the largest memory allocation request ** handed to [pagecache memory allocator]. Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
)^ ** -** ^(
SQLITE_STATUS_SCRATCH_USED
+** [[SQLITE_STATUS_SCRATCH_USED]] ^(
SQLITE_STATUS_SCRATCH_USED
**
This parameter returns the number of allocations used out of the ** [scratch memory allocator] configured using ** [SQLITE_CONFIG_SCRATCH]. The value returned is in allocations, not @@ -6049,7 +6215,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** outstanding at time, this parameter also reports the number of threads ** using scratch memory at the same time.
)^ ** -** ^(
SQLITE_STATUS_SCRATCH_OVERFLOW
+** [[SQLITE_STATUS_SCRATCH_OVERFLOW]] ^(
SQLITE_STATUS_SCRATCH_OVERFLOW
**
This parameter returns the number of bytes of scratch memory ** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] ** buffer and where forced to overflow to [sqlite3_malloc()]. The values @@ -6059,13 +6225,13 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** slots were available. **
)^ ** -** ^(
SQLITE_STATUS_SCRATCH_SIZE
+** [[SQLITE_STATUS_SCRATCH_SIZE]] ^(
SQLITE_STATUS_SCRATCH_SIZE
**
This parameter records the largest memory allocation request ** handed to [scratch memory allocator]. Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
)^ ** -** ^(
SQLITE_STATUS_PARSER_STACK
+** [[SQLITE_STATUS_PARSER_STACK]] ^(
SQLITE_STATUS_PARSER_STACK
**
This parameter records the deepest parser stack. It is only ** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
)^ **
@@ -6090,9 +6256,9 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** about a single [database connection]. ^The first argument is the ** database connection object to be interrogated. ^The second argument ** is an integer constant, taken from the set of -** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros, that +** [SQLITE_DBSTATUS options], that ** determines the parameter to interrogate. The set of -** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros is likely +** [SQLITE_DBSTATUS options] is likely ** to grow in future releases of SQLite. ** ** ^The current value of the requested parameter is written into *pCur @@ -6109,6 +6275,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r /* ** CAPI3REF: Status Parameters for database connections +** KEYWORDS: {SQLITE_DBSTATUS options} ** ** These constants are the available integer "verbs" that can be passed as ** the second argument to the [sqlite3_db_status()] interface. @@ -6120,15 +6287,16 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** if a discontinued or unsupported verb is invoked. ** **
-** ^(
SQLITE_DBSTATUS_LOOKASIDE_USED
+** [[SQLITE_DBSTATUS_LOOKASIDE_USED]] ^(
SQLITE_DBSTATUS_LOOKASIDE_USED
**
This parameter returns the number of lookaside memory slots currently ** checked out.
)^ ** -** ^(
SQLITE_DBSTATUS_LOOKASIDE_HIT
+** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
SQLITE_DBSTATUS_LOOKASIDE_HIT
**
This parameter returns the number malloc attempts that were ** satisfied using lookaside memory. Only the high-water value is meaningful; ** the current value is always zero.)^ ** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]] ** ^(
SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE
**
This parameter returns the number malloc attempts that might have ** been satisfied using lookaside memory but failed due to the amount of @@ -6136,6 +6304,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** Only the high-water value is meaningful; ** the current value is always zero.)^ ** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]] ** ^(
SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL
**
This parameter returns the number malloc attempts that might have ** been satisfied using lookaside memory but failed due to all lookaside @@ -6143,12 +6312,12 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** Only the high-water value is meaningful; ** the current value is always zero.)^ ** -** ^(
SQLITE_DBSTATUS_CACHE_USED
+** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
SQLITE_DBSTATUS_CACHE_USED
**
This parameter returns the approximate number of of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. ** -** ^(
SQLITE_DBSTATUS_SCHEMA_USED
+** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
SQLITE_DBSTATUS_SCHEMA_USED
**
This parameter returns the approximate number of of bytes of heap ** memory used to store the schema for all databases associated ** with the connection - main, temp, and any [ATTACH]-ed databases.)^ @@ -6157,7 +6326,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** [shared cache mode] being enabled. ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. ** -** ^(
SQLITE_DBSTATUS_STMT_USED
+** [[SQLITE_DBSTATUS_STMT_USED]] ^(
SQLITE_DBSTATUS_STMT_USED
**
This parameter returns the approximate number of of bytes of heap ** and lookaside memory used by all prepared statements associated with ** the database connection.)^ @@ -6179,7 +6348,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** CAPI3REF: Prepared Statement Status ** ** ^(Each prepared statement maintains various -** [SQLITE_STMTSTATUS_SORT | counters] that measure the number +** [SQLITE_STMTSTATUS counters] that measure the number ** of times it has performed specific operations.)^ These counters can ** be used to monitor the performance characteristics of the prepared ** statements. For example, if the number of table steps greatly exceeds @@ -6190,7 +6359,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** ^(This interface is used to retrieve and reset counter values from ** a [prepared statement]. The first argument is the prepared statement ** object to be interrogated. The second argument -** is an integer code for a specific [SQLITE_STMTSTATUS_SORT | counter] +** is an integer code for a specific [SQLITE_STMTSTATUS counter] ** to be interrogated.)^ ** ^The current value of the requested counter is returned. ** ^If the resetFlg is true, then the counter is reset to zero after this @@ -6202,24 +6371,25 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); /* ** CAPI3REF: Status Parameters for prepared statements +** KEYWORDS: {SQLITE_STMTSTATUS counter} {SQLITE_STMTSTATUS counters} ** ** These preprocessor macros define integer codes that name counter ** values associated with the [sqlite3_stmt_status()] interface. ** The meanings of the various counters are as follows: ** **
-**
SQLITE_STMTSTATUS_FULLSCAN_STEP
+** [[SQLITE_STMTSTATUS_FULLSCAN_STEP]]
SQLITE_STMTSTATUS_FULLSCAN_STEP
**
^This is the number of times that SQLite has stepped forward in ** a table as part of a full table scan. Large numbers for this counter ** may indicate opportunities for performance improvement through ** careful use of indices.
** -**
SQLITE_STMTSTATUS_SORT
+** [[SQLITE_STMTSTATUS_SORT]]
SQLITE_STMTSTATUS_SORT
**
^This is the number of sort operations that have occurred. ** A non-zero value in this counter may indicate an opportunity to ** improvement performance through careful use of indices.
** -**
SQLITE_STMTSTATUS_AUTOINDEX
+** [[SQLITE_STMTSTATUS_AUTOINDEX]]
SQLITE_STMTSTATUS_AUTOINDEX
**
^This is the number of rows inserted into transient indices that ** were created automatically in order to help joins run faster. ** A non-zero value in this counter may indicate an opportunity to @@ -6270,6 +6440,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** the application may discard the parameter after the call to ** [sqlite3_config()] returns.)^ ** +** [[the xInit() page cache method]] ** ^(The xInit() method is called once for each effective ** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() @@ -6280,6 +6451,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** built-in default page cache is used instead of the application defined ** page cache.)^ ** +** [[the xShutdown() page cache method]] ** ^The xShutdown() method is called by [sqlite3_shutdown()]. ** It can be used to clean up ** any outstanding resources before process shutdown, if required. @@ -6294,6 +6466,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^SQLite will never invoke xInit() more than once without an intervening ** call to xShutdown(). ** +** [[the xCreate() page cache methods]] ** ^SQLite invokes the xCreate() method to construct a new cache instance. ** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The @@ -6318,6 +6491,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^Hence, a cache created with bPurgeable false will ** never contain any unpinned pages. ** +** [[the xCachesize() page cache method]] ** ^(The xCachesize() method may be called at any time by SQLite to set the ** suggested maximum cache-size (number of pages stored by) the cache ** instance passed as the first argument. This is the value configured using @@ -6325,14 +6499,16 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** parameter, the implementation is not required to do anything with this ** value; it is advisory only. ** +** [[the xPagecount() page cache methods]] ** The xPagecount() method must return the number of pages currently ** stored in the cache, both pinned and unpinned. ** +** [[the xFetch() page cache methods]] ** The xFetch() method locates a page in the cache and returns a pointer to ** the page, or a NULL pointer. ** A "page", in this context, means a buffer of szPage bytes aligned at an ** 8-byte boundary. The page to be fetched is determined by the key. ^The -** mimimum key value is 1. After it has been retrieved using xFetch, the page +** minimum key value is 1. After it has been retrieved using xFetch, the page ** is considered to be "pinned". ** ** If the requested page is already in the page cache, then the page cache @@ -6356,6 +6532,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** attempt to unpin one or more cache pages by spilling the content of ** pinned pages to disk and synching the operating system disk cache. ** +** [[the xUnpin() page cache method]] ** ^xUnpin() is called by SQLite with a pointer to a currently pinned page ** as its second argument. If the third parameter, discard, is non-zero, ** then the page must be evicted from the cache. @@ -6368,6 +6545,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** call to xUnpin() unpins the page regardless of the number of prior calls ** to xFetch(). ** +** [[the xRekey() page cache methods]] ** The xRekey() method is used to change the key value associated with the ** page passed as the second argument. If the cache ** previously contains an entry associated with newKey, it must be @@ -6380,6 +6558,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** of these pages are pinned, they are implicitly unpinned, meaning that ** they can be safely discarded. ** +** [[the xDestroy() page cache method]] ** ^The xDestroy() method is used to delete a cache allocated by xCreate(). ** All resources associated with the specified cache should be freed. ^After ** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*] @@ -6442,7 +6621,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** There should be exactly one call to sqlite3_backup_finish() for each ** successful call to sqlite3_backup_init(). ** -** sqlite3_backup_init() +** [[sqlite3_backup_init()]] sqlite3_backup_init() ** ** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the ** [database connection] associated with the destination database @@ -6469,7 +6648,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** sqlite3_backup_finish() functions to perform the specified backup ** operation. ** -** sqlite3_backup_step() +** [[sqlite3_backup_step()]] sqlite3_backup_step() ** ** ^Function sqlite3_backup_step(B,N) will copy up to N pages between ** the source and destination databases specified by [sqlite3_backup] object B. @@ -6526,7 +6705,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** by the backup operation, then the backup database is automatically ** updated at the same time. ** -** sqlite3_backup_finish() +** [[sqlite3_backup_finish()]] sqlite3_backup_finish() ** ** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the ** application wishes to abandon the backup operation, the application @@ -6549,7 +6728,8 @@ typedef struct sqlite3_backup sqlite3_backup; ** is not a permanent error and does not affect the return value of ** sqlite3_backup_finish(). ** -** sqlite3_backup_remaining(), sqlite3_backup_pagecount() +** [[sqlite3_backup__remaining()]] [[sqlite3_backup_pagecount()]] +** sqlite3_backup_remaining() and sqlite3_backup_pagecount() ** ** ^Each call to sqlite3_backup_step() sets two values inside ** the [sqlite3_backup] object: the number of pages still to be backed @@ -6935,6 +7115,93 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( #define SQLITE_CHECKPOINT_FULL 1 #define SQLITE_CHECKPOINT_RESTART 2 +/* +** CAPI3REF: Virtual Table Interface Configuration +** +** This function may be called by either the [xConnect] or [xCreate] method +** of a [virtual table] implementation to configure +** various facets of the virtual table interface. +** +** If this interface is invoked outside the context of an xConnect or +** xCreate virtual table method then the behavior is undefined. +** +** At present, there is only one option that may be configured using +** this function. (See [SQLITE_VTAB_CONSTRAINT_SUPPORT].) Further options +** may be added in the future. +*/ +SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Virtual Table Configuration Options +** +** These macros define the various options to the +** [sqlite3_vtab_config()] interface that [virtual table] implementations +** can use to customize and optimize their behavior. +** +**
+**
SQLITE_VTAB_CONSTRAINT_SUPPORT +**
Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported, +** where X is an integer. If X is zero, then the [virtual table] whose +** [xCreate] or [xConnect] method invoked [sqlite3_vtab_config()] does not +** support constraints. In this configuration (which is the default) if +** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire +** statement is rolled back as if [ON CONFLICT | OR ABORT] had been +** specified as part of the users SQL statement, regardless of the actual +** ON CONFLICT mode specified. +** +** If X is non-zero, then the virtual table implementation guarantees +** that if [xUpdate] returns [SQLITE_CONSTRAINT], it will do so before +** any modifications to internal or persistent data structures have been made. +** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite +** is able to roll back a statement or database transaction, and abandon +** or continue processing the current SQL statement as appropriate. +** If the ON CONFLICT mode is REPLACE and the [xUpdate] method returns +** [SQLITE_CONSTRAINT], SQLite handles this as if the ON CONFLICT mode +** had been ABORT. +** +** Virtual table implementations that are required to handle OR REPLACE +** must do so within the [xUpdate] method. If a call to the +** [sqlite3_vtab_on_conflict()] function indicates that the current ON +** CONFLICT policy is REPLACE, the virtual table implementation should +** silently replace the appropriate rows within the xUpdate callback and +** return SQLITE_OK. Or, if this is not possible, it may return +** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT +** constraint handling. +**
+*/ +#define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 + +/* +** CAPI3REF: Determine The Virtual Table Conflict Policy +** +** This function may only be called from within a call to the [xUpdate] method +** of a [virtual table] implementation for an INSERT or UPDATE operation. ^The +** value returned is one of [SQLITE_ROLLBACK], [SQLITE_IGNORE], [SQLITE_FAIL], +** [SQLITE_ABORT], or [SQLITE_REPLACE], according to the [ON CONFLICT] mode +** of the SQL statement that triggered the call to the [xUpdate] method of the +** [virtual table]. +*/ +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); + +/* +** CAPI3REF: Conflict resolution modes +** +** These constants are returned by [sqlite3_vtab_on_conflict()] to +** inform a [virtual table] implementation what the [ON CONFLICT] mode +** is for the SQL statement being evaluated. +** +** Note that the [SQLITE_IGNORE] constant is also used as a potential +** return value from the [sqlite3_set_authorizer()] callback and that +** [SQLITE_ABORT] is also a [result code]. +*/ +#define SQLITE_ROLLBACK 1 +/* #define SQLITE_IGNORE 2 // Also used by sqlite3_authorizer() callback */ +#define SQLITE_FAIL 3 +/* #define SQLITE_ABORT 4 // Also an error code */ +#define SQLITE_REPLACE 5 + + /* ** Undo the hack that converts floating point types to integer for @@ -7601,6 +7868,7 @@ typedef struct TriggerPrg TriggerPrg; typedef struct TriggerStep TriggerStep; typedef struct UnpackedRecord UnpackedRecord; typedef struct VTable VTable; +typedef struct VtabCtx VtabCtx; typedef struct Walker Walker; typedef struct WherePlan WherePlan; typedef struct WhereInfo WhereInfo; @@ -7657,6 +7925,7 @@ typedef struct BtShared BtShared; SQLITE_PRIVATE int sqlite3BtreeOpen( + sqlite3_vfs *pVfs, /* VFS to use with this b-tree */ const char *zFilename, /* Name of database file to open */ sqlite3 *db, /* Associated database connection */ Btree **ppBtree, /* Return open Btree* here */ @@ -8217,6 +8486,7 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp); +SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); @@ -8230,7 +8500,7 @@ SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDeleteObject(sqlite3*,Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,int,int,int,int,int,int); +SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe*, int); SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe*); @@ -8239,6 +8509,7 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *, int); SQLITE_PRIVATE void sqlite3VdbeTrace(Vdbe*,FILE*); #endif SQLITE_PRIVATE void sqlite3VdbeResetStepResult(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe*); SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe*,int); SQLITE_PRIVATE int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*)); @@ -9015,7 +9286,7 @@ struct Db { ** A thread must be holding a mutex on the corresponding Btree in order ** to access Schema content. This implies that the thread must also be ** holding a mutex on the sqlite3 connection pointer that owns the Btree. -** For a TEMP Schema, on the connection mutex is required. +** For a TEMP Schema, only the connection mutex is required. */ struct Schema { int schema_cookie; /* Database schema version number for this file */ @@ -9136,7 +9407,7 @@ struct sqlite3 { int nDb; /* Number of backends currently in use */ Db *aDb; /* All backends */ int flags; /* Miscellaneous flags. See below */ - int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ + unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ u8 autoCommit; /* The auto-commit flag. */ @@ -9145,6 +9416,7 @@ struct sqlite3 { u8 dfltLockMode; /* Default locking-mode for attached dbs */ signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ u8 suppressErr; /* Do not issue error messages if true */ + u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ int nextPagesize; /* Pagesize after VACUUM if >0 */ int nTable; /* Number of tables in the database */ CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ @@ -9203,7 +9475,7 @@ struct sqlite3 { #endif #ifndef SQLITE_OMIT_VIRTUALTABLE Hash aModule; /* populated by sqlite3_create_module() */ - Table *pVTab; /* vtab with active Connect/Create method */ + VtabCtx *pVtabCtx; /* Context for active vtab connect/create */ VTable **aVTrans; /* Virtual tables with open transactions */ int nVTrans; /* Allocated size of aVTrans */ VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */ @@ -9287,6 +9559,7 @@ struct sqlite3 { #define SQLITE_IndexCover 0x10 /* Disable index covering table */ #define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */ +#define SQLITE_IdxRealAsInt 0x80 /* Store REAL as INT in indices */ #define SQLITE_OptMask 0xff /* Mask of all disablable opts */ /* @@ -9566,6 +9839,8 @@ struct VTable { Module *pMod; /* Pointer to module implementation */ sqlite3_vtab *pVtab; /* Pointer to vtab instance */ int nRef; /* Number of pointers to this structure */ + u8 bConstraint; /* True if constraints are supported */ + int iSavepoint; /* Depth of the SAVEPOINT stack */ VTable *pNext; /* Next in linked list (see above) */ }; @@ -10560,9 +10835,8 @@ struct Parse { ** each recursion */ int nVar; /* Number of '?' variables seen in the SQL so far */ - int nVarExpr; /* Number of used slots in apVarExpr[] */ - int nVarExprAlloc; /* Number of allocated slots in apVarExpr[] */ - Expr **apVarExpr; /* Pointers to :aaa and $aaaa wildcard expressions */ + int nzVar; /* Number of available slots in azVar[] */ + char **azVar; /* Pointers to names of parameters */ Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ int nAlias; /* Number of aliased result set columns */ int nAliasAlloc; /* Number of allocated slots for aAlias[] */ @@ -10754,6 +11028,7 @@ struct Sqlite3Config { int bMemstat; /* True to enable memory status */ int bCoreMutex; /* True to enable core mutexing */ int bFullMutex; /* True to enable full mutexing */ + int bOpenUri; /* True to interpret filenames as URIs */ int mxStrlen; /* Maximum string length */ int szLookaside; /* Default lookaside buffer size */ int nLookaside; /* Default lookaside buffer count */ @@ -10782,6 +11057,7 @@ struct Sqlite3Config { int nRefInitMutex; /* Number of users of pInitMutex */ void (*xLog)(void*,int,const char*); /* Function for logging */ void *pLogArg; /* First argument to xLog() */ + int bLocaltimeFault; /* True to fail localtime() calls */ }; /* @@ -11003,6 +11279,8 @@ SQLITE_PRIVATE void sqlite3AddColumnType(Parse*,Token*); SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse*,ExprSpan*); SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*); SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,Select*); +SQLITE_PRIVATE int sqlite3ParseUri(const char*,const char*,unsigned int*, + sqlite3_vfs**,char**,char **); SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32); SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec*, u32); @@ -11207,7 +11485,7 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); SQLITE_PRIVATE int sqlite3Atoi(const char*); SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar); SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte); -SQLITE_PRIVATE int sqlite3Utf8Read(const u8*, const u8**); +SQLITE_PRIVATE u32 sqlite3Utf8Read(const u8*, const u8**); /* ** Routines to read and write variable-length integers. These used to @@ -11253,6 +11531,7 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr); SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); SQLITE_PRIVATE void sqlite3Error(sqlite3*, int, const char*,...); SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); +SQLITE_PRIVATE u8 sqlite3HexToInt(int h); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); SQLITE_PRIVATE const char *sqlite3ErrStr(int); SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); @@ -11268,6 +11547,12 @@ SQLITE_PRIVATE int sqlite3AddInt64(i64*,i64); SQLITE_PRIVATE int sqlite3SubInt64(i64*,i64); SQLITE_PRIVATE int sqlite3MulInt64(i64*,i64); SQLITE_PRIVATE int sqlite3AbsInt32(int); +#ifdef SQLITE_ENABLE_8_3_NAMES +SQLITE_PRIVATE void sqlite3FileSuffix3(const char*, char*); +#else +# define sqlite3FileSuffix3(X,Y) +#endif +SQLITE_PRIVATE u8 sqlite3GetBoolean(const char *z); SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value*, u8); SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value*, u8); @@ -11377,6 +11662,7 @@ SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char*); # define sqlite3VtabLock(X) # define sqlite3VtabUnlock(X) # define sqlite3VtabUnlockList(X) +# define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK #else SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table*); SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, char **); @@ -11385,6 +11671,7 @@ SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db); SQLITE_PRIVATE void sqlite3VtabLock(VTable *); SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *); SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3*); +SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *, int, int); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse*,Table*); @@ -11691,7 +11978,9 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { }; #endif - +#ifndef SQLITE_USE_URI +# define SQLITE_USE_URI 0 +#endif /* ** The following singleton contains the global configuration for @@ -11701,6 +11990,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { SQLITE_DEFAULT_MEMSTATUS, /* bMemstat */ 1, /* bCoreMutex */ SQLITE_THREADSAFE==1, /* bFullMutex */ + SQLITE_USE_URI, /* bOpenUri */ 0x7ffffffe, /* mxStrlen */ 100, /* szLookaside */ 500, /* nLookaside */ @@ -11728,6 +12018,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nRefInitMutex */ 0, /* xLog */ 0, /* pLogArg */ + 0, /* bLocaltimeFault */ }; @@ -12480,11 +12771,11 @@ struct Vdbe { Mem *aVar; /* Values for the OP_Variable opcode. */ char **azVar; /* Name of variables */ ynVar nVar; /* Number of entries in aVar[] */ + ynVar nzVar; /* Number of entries in azVar[] */ u32 cacheCtr; /* VdbeCursor row cache generation counter */ int pc; /* The program counter */ int rc; /* Value to return */ u8 errorAction; /* Recovery action to do in case of an error */ - u8 okVar; /* True if azVar[] has been initialized */ u8 explain; /* True if EXPLAIN present on SQL command */ u8 changeCntOn; /* True to update the change-counter */ u8 expired; /* True if the VM needs to be recompiled */ @@ -12878,22 +13169,6 @@ SQLITE_API int sqlite3_db_status( #ifndef SQLITE_OMIT_DATETIME_FUNCS -/* -** On recent Windows platforms, the localtime_s() function is available -** as part of the "Secure CRT". It is essentially equivalent to -** localtime_r() available under most POSIX platforms, except that the -** order of the parameters is reversed. -** -** See http://msdn.microsoft.com/en-us/library/a442x3ye(VS.80).aspx. -** -** If the user has not indicated to use localtime_r() or localtime_s() -** already, check for an MSVC build environment that provides -** localtime_s(). -*/ -#if !defined(HAVE_LOCALTIME_R) && !defined(HAVE_LOCALTIME_S) && \ - defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) -#define HAVE_LOCALTIME_S 1 -#endif /* ** A structure for holding a single date and time. @@ -13239,15 +13514,83 @@ static void clearYMD_HMS_TZ(DateTime *p){ p->validTZ = 0; } +/* +** On recent Windows platforms, the localtime_s() function is available +** as part of the "Secure CRT". It is essentially equivalent to +** localtime_r() available under most POSIX platforms, except that the +** order of the parameters is reversed. +** +** See http://msdn.microsoft.com/en-us/library/a442x3ye(VS.80).aspx. +** +** If the user has not indicated to use localtime_r() or localtime_s() +** already, check for an MSVC build environment that provides +** localtime_s(). +*/ +#if !defined(HAVE_LOCALTIME_R) && !defined(HAVE_LOCALTIME_S) && \ + defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) +#define HAVE_LOCALTIME_S 1 +#endif + #ifndef SQLITE_OMIT_LOCALTIME /* -** Compute the difference (in milliseconds) -** between localtime and UTC (a.k.a. GMT) -** for the time value p where p is in UTC. +** The following routine implements the rough equivalent of localtime_r() +** using whatever operating-system specific localtime facility that +** is available. This routine returns 0 on success and +** non-zero on any kind of error. +** +** If the sqlite3GlobalConfig.bLocaltimeFault variable is true then this +** routine will always fail. */ -static sqlite3_int64 localtimeOffset(DateTime *p){ +static int osLocaltime(time_t *t, struct tm *pTm){ + int rc; +#if (!defined(HAVE_LOCALTIME_R) || !HAVE_LOCALTIME_R) \ + && (!defined(HAVE_LOCALTIME_S) || !HAVE_LOCALTIME_S) + struct tm *pX; + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex_enter(mutex); + pX = localtime(t); +#ifndef SQLITE_OMIT_BUILTIN_TEST + if( sqlite3GlobalConfig.bLocaltimeFault ) pX = 0; +#endif + if( pX ) *pTm = *pX; + sqlite3_mutex_leave(mutex); + rc = pX==0; +#else +#ifndef SQLITE_OMIT_BUILTIN_TEST + if( sqlite3GlobalConfig.bLocaltimeFault ) return 1; +#endif +#if defined(HAVE_LOCALTIME_R) && HAVE_LOCALTIME_R + rc = localtime_r(t, pTm)==0; +#else + rc = localtime_s(pTm, t); +#endif /* HAVE_LOCALTIME_R */ +#endif /* HAVE_LOCALTIME_R || HAVE_LOCALTIME_S */ + return rc; +} +#endif /* SQLITE_OMIT_LOCALTIME */ + + +#ifndef SQLITE_OMIT_LOCALTIME +/* +** Compute the difference (in milliseconds) between localtime and UTC +** (a.k.a. GMT) for the time value p where p is in UTC. If no error occurs, +** return this value and set *pRc to SQLITE_OK. +** +** Or, if an error does occur, set *pRc to SQLITE_ERROR. The returned value +** is undefined in this case. +*/ +static sqlite3_int64 localtimeOffset( + DateTime *p, /* Date at which to calculate offset */ + sqlite3_context *pCtx, /* Write error here if one occurs */ + int *pRc /* OUT: Error code. SQLITE_OK or ERROR */ +){ DateTime x, y; time_t t; + struct tm sLocal; + + /* Initialize the contents of sLocal to avoid a compiler warning. */ + memset(&sLocal, 0, sizeof(sLocal)); + x = *p; computeYMD_HMS(&x); if( x.Y<1971 || x.Y>=2038 ){ @@ -13265,47 +13608,23 @@ static sqlite3_int64 localtimeOffset(DateTime *p){ x.validJD = 0; computeJD(&x); t = (time_t)(x.iJD/1000 - 21086676*(i64)10000); -#ifdef HAVE_LOCALTIME_R - { - struct tm sLocal; - localtime_r(&t, &sLocal); - y.Y = sLocal.tm_year + 1900; - y.M = sLocal.tm_mon + 1; - y.D = sLocal.tm_mday; - y.h = sLocal.tm_hour; - y.m = sLocal.tm_min; - y.s = sLocal.tm_sec; + if( osLocaltime(&t, &sLocal) ){ + sqlite3_result_error(pCtx, "local time unavailable", -1); + *pRc = SQLITE_ERROR; + return 0; } -#elif defined(HAVE_LOCALTIME_S) && HAVE_LOCALTIME_S - { - struct tm sLocal; - localtime_s(&sLocal, &t); - y.Y = sLocal.tm_year + 1900; - y.M = sLocal.tm_mon + 1; - y.D = sLocal.tm_mday; - y.h = sLocal.tm_hour; - y.m = sLocal.tm_min; - y.s = sLocal.tm_sec; - } -#else - { - struct tm *pTm; - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); - pTm = localtime(&t); - y.Y = pTm->tm_year + 1900; - y.M = pTm->tm_mon + 1; - y.D = pTm->tm_mday; - y.h = pTm->tm_hour; - y.m = pTm->tm_min; - y.s = pTm->tm_sec; - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); - } -#endif + y.Y = sLocal.tm_year + 1900; + y.M = sLocal.tm_mon + 1; + y.D = sLocal.tm_mday; + y.h = sLocal.tm_hour; + y.m = sLocal.tm_min; + y.s = sLocal.tm_sec; y.validYMD = 1; y.validHMS = 1; y.validJD = 0; y.validTZ = 0; computeJD(&y); + *pRc = SQLITE_OK; return y.iJD - x.iJD; } #endif /* SQLITE_OMIT_LOCALTIME */ @@ -13329,9 +13648,12 @@ static sqlite3_int64 localtimeOffset(DateTime *p){ ** localtime ** utc ** -** Return 0 on success and 1 if there is any kind of error. +** Return 0 on success and 1 if there is any kind of error. If the error +** is in a system call (i.e. localtime()), then an error message is written +** to context pCtx. If the error is an unrecognized modifier, no error is +** written to pCtx. */ -static int parseModifier(const char *zMod, DateTime *p){ +static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ int rc = 1; int n; double r; @@ -13351,9 +13673,8 @@ static int parseModifier(const char *zMod, DateTime *p){ */ if( strcmp(z, "localtime")==0 ){ computeJD(p); - p->iJD += localtimeOffset(p); + p->iJD += localtimeOffset(p, pCtx, &rc); clearYMD_HMS_TZ(p); - rc = 0; } break; } @@ -13374,11 +13695,12 @@ static int parseModifier(const char *zMod, DateTime *p){ else if( strcmp(z, "utc")==0 ){ sqlite3_int64 c1; computeJD(p); - c1 = localtimeOffset(p); - p->iJD -= c1; - clearYMD_HMS_TZ(p); - p->iJD += c1 - localtimeOffset(p); - rc = 0; + c1 = localtimeOffset(p, pCtx, &rc); + if( rc==SQLITE_OK ){ + p->iJD -= c1; + clearYMD_HMS_TZ(p); + p->iJD += c1 - localtimeOffset(p, pCtx, &rc); + } } #endif break; @@ -13559,9 +13881,8 @@ static int isDate( } } for(i=1; i= mem0.alarmThreshold ){ + if( nUsed >= mem0.alarmThreshold - nFull ){ mem0.nearlyFull = 1; sqlite3MallocAlarm(nFull); }else{ @@ -18191,7 +18512,7 @@ SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ ** Change the size of an existing memory allocation */ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ - int nOld, nNew; + int nOld, nNew, nDiff; void *pNew; if( pOld==0 ){ return sqlite3Malloc(nBytes); /* IMP: R-28354-25769 */ @@ -18214,9 +18535,10 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, nBytes); - if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nNew-nOld >= - mem0.alarmThreshold ){ - sqlite3MallocAlarm(nNew-nOld); + nDiff = nNew - nOld; + if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= + mem0.alarmThreshold-nDiff ){ + sqlite3MallocAlarm(nDiff); } assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); @@ -19801,7 +20123,7 @@ static const unsigned char sqlite3Utf8Trans1[] = { || (c&0xFFFFF800)==0xD800 \ || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ } -SQLITE_PRIVATE int sqlite3Utf8Read( +SQLITE_PRIVATE u32 sqlite3Utf8Read( const unsigned char *zIn, /* First byte of UTF-8 character */ const unsigned char **pzNext /* Write first byte past UTF-8 char here */ ){ @@ -21182,13 +21504,12 @@ SQLITE_PRIVATE void sqlite3Put4byte(unsigned char *p, u32 v){ -#if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) /* ** Translate a single byte of Hex into an integer. ** This routine only works if h really is a valid hexadecimal ** character: 0..9a..fA..F */ -static u8 hexToInt(int h){ +SQLITE_PRIVATE u8 sqlite3HexToInt(int h){ assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); #ifdef SQLITE_ASCII h += 9*(1&(h>>6)); @@ -21198,7 +21519,6 @@ static u8 hexToInt(int h){ #endif return (u8)(h & 0xf); } -#endif /* !SQLITE_OMIT_BLOB_LITERAL || SQLITE_HAS_CODEC */ #if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) /* @@ -21215,7 +21535,7 @@ SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ n--; if( zBlob ){ for(i=0; i test.nal +** test.db-wal => test.wal +** test.db-shm => test.shm +*/ +SQLITE_PRIVATE void sqlite3FileSuffix3(const char *zBaseFilename, char *z){ + const char *zOk; + zOk = sqlite3_uri_parameter(zBaseFilename, "8_3_names"); + if( zOk && sqlite3GetBoolean(zOk) ){ + int i, sz; + sz = sqlite3Strlen30(z); + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} + if( z[i]=='.' && ALWAYS(sz>i+4) ) memcpy(&z[i+1], &z[sz-3], 4); + } +} +#endif + /************** End of util.c ************************************************/ /************** Begin file hash.c ********************************************/ /* @@ -24054,6 +24400,10 @@ SQLITE_API int sqlite3_os_end(void){ # include #endif +#ifdef HAVE_UTIME +# include +#endif + /* ** Allowed values of unixFile.fsFlags */ @@ -24401,6 +24751,18 @@ SQLITE_API int sqlite3_open_file_count = 0; #define threadid 0 #endif +/* +** Different Unix systems declare open() in different ways. Same use +** open(const char*,int,mode_t). Others use open(const char*,int,...). +** The difference is important when using a pointer to the function. +** +** The safest way to deal with the problem is to always use this wrapper +** which always has the same well-defined interface. +*/ +static int posixOpen(const char *zFile, int flags, int mode){ + return open(zFile, flags, mode); +} + /* ** Many system calls are accessed through pointer-to-functions so that ** they may be overridden at runtime to facilitate fault injection during @@ -24412,8 +24774,8 @@ static struct unix_syscall { sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { - { "open", (sqlite3_syscall_ptr)open, 0 }, -#define osOpen ((int(*)(const char*,int,...))aSyscall[0].pCurrent) + { "open", (sqlite3_syscall_ptr)posixOpen, 0 }, +#define osOpen ((int(*)(const char*,int,int))aSyscall[0].pCurrent) { "close", (sqlite3_syscall_ptr)close, 0 }, #define osClose ((int(*)(int))aSyscall[1].pCurrent) @@ -24450,7 +24812,7 @@ static struct unix_syscall { { "read", (sqlite3_syscall_ptr)read, 0 }, #define osRead ((ssize_t(*)(int,void*,size_t))aSyscall[8].pCurrent) -#if defined(USE_PREAD) || defined(SQLITE_ENABLE_LOCKING_STYLE) +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE { "pread", (sqlite3_syscall_ptr)pread, 0 }, #else { "pread", (sqlite3_syscall_ptr)0, 0 }, @@ -24467,7 +24829,7 @@ static struct unix_syscall { { "write", (sqlite3_syscall_ptr)write, 0 }, #define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) -#if defined(USE_PREAD) || defined(SQLITE_ENABLE_LOCKING_STYLE) +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE { "pwrite", (sqlite3_syscall_ptr)pwrite, 0 }, #else { "pwrite", (sqlite3_syscall_ptr)0, 0 }, @@ -25055,7 +25417,7 @@ struct unixInodeInfo { UnixUnusedFd *pUnused; /* Unused file descriptors to close */ unixInodeInfo *pNext; /* List of all unixInodeInfo objects */ unixInodeInfo *pPrev; /* .... doubly linked */ -#if defined(SQLITE_ENABLE_LOCKING_STYLE) +#if SQLITE_ENABLE_LOCKING_STYLE unsigned long long sharedByte; /* for AFP simulated shared lock */ #endif #if OS_VXWORKS @@ -26049,8 +26411,10 @@ static int dotlockLock(sqlite3_file *id, int eFileLock) { */ if( pFile->eFileLock > NO_LOCK ){ pFile->eFileLock = eFileLock; -#if !OS_VXWORKS && !__minix /* Always update the timestamp on the old file */ +#ifdef HAVE_UTIME + utime(zLockFile, NULL); +#else utimes(zLockFile, NULL); #endif return SQLITE_OK; @@ -27212,7 +27576,7 @@ static int unixWrite( SimulateDiskfullError(( wrote=0, amt=1 )); if( amt>0 ){ - if( wrote<0 ){ + if( wrote<0 && pFile->lastErrno!=ENOSPC ){ /* lastErrno set by seekAndWrite */ return SQLITE_IOERR_WRITE; }else{ @@ -27651,7 +28015,8 @@ struct unixShmNode { char *zFilename; /* Name of the mmapped file */ int h; /* Open file descriptor */ int szRegion; /* Size of shared-memory regions */ - int nRegion; /* Size of array apRegion */ + u16 nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ char **apRegion; /* Array of mapped shared-memory regions */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ @@ -27883,6 +28248,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ (u32)sStat.st_ino, (u32)sStat.st_dev); #else sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", pDbFd->zPath); + sqlite3FileSuffix3(pDbFd->zPath, zShmFilename); #endif pShmNode->h = -1; pDbFd->pInode->pShmNode = pShmNode; @@ -27897,8 +28263,17 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ pShmNode->h = robust_open(zShmFilename, O_RDWR|O_CREAT, (sStat.st_mode & 0777)); if( pShmNode->h<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); - goto shm_open_err; + const char *zRO; + zRO = sqlite3_uri_parameter(pDbFd->zPath, "readonly_shm"); + if( zRO && sqlite3GetBoolean(zRO) ){ + pShmNode->h = robust_open(zShmFilename, O_RDONLY, + (sStat.st_mode & 0777)); + pShmNode->isReadonly = 1; + } + if( pShmNode->h<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); + goto shm_open_err; + } } /* Check to see if another process is holding the dead-man switch. @@ -28037,11 +28412,12 @@ static int unixShmMap( while(pShmNode->nRegion<=iRegion){ void *pMem; if( pShmNode->h>=0 ){ - pMem = mmap(0, szRegion, PROT_READ|PROT_WRITE, + pMem = mmap(0, szRegion, + pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, pShmNode->nRegion*szRegion ); if( pMem==MAP_FAILED ){ - rc = SQLITE_IOERR; + rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); goto shmpage_out; } }else{ @@ -28063,6 +28439,7 @@ shmpage_out: }else{ *pp = 0; } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; sqlite3_mutex_leave(pShmNode->mutex); return rc; } @@ -28916,6 +29293,11 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** corresponding database file and sets *pMode to this value. Whenever ** possible, WAL and journal files are created using the same permissions ** as the associated database file. +** +** If the SQLITE_ENABLE_8_3_NAMES option is enabled, then the +** original filename is unavailable. But 8_3_NAMES is only used for +** FAT filesystems and permissions do not matter there, so just use +** the default permissions. */ static int findCreateFileMode( const char *zPath, /* Path of file (possibly) being created */ @@ -28923,6 +29305,7 @@ static int findCreateFileMode( mode_t *pMode /* OUT: Permissions to open file with */ ){ int rc = SQLITE_OK; /* Return Code */ + *pMode = SQLITE_DEFAULT_FILE_PERMISSIONS; if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ char zDb[MAX_PATHNAME+1]; /* Database file path */ int nDb; /* Number of valid bytes in zDb */ @@ -28934,15 +29317,15 @@ static int findCreateFileMode( ** ** "-journal" ** "-wal" - ** "-journal-NNNN" - ** "-wal-NNNN" + ** "-journalNN" + ** "-walNN" ** - ** where NNNN is a 4 digit decimal number. The NNNN naming schemes are + ** where NN is a 4 digit decimal number. The NN naming schemes are ** used by the test_multiplex.c module. */ nDb = sqlite3Strlen30(zPath) - 1; - while( nDb>0 && zPath[nDb]!='l' ) nDb--; - nDb -= ((flags & SQLITE_OPEN_WAL) ? 3 : 7); + while( nDb>0 && zPath[nDb]!='-' ) nDb--; + if( nDb==0 ) return SQLITE_OK; memcpy(zDb, zPath, nDb); zDb[nDb] = '\0'; @@ -28953,8 +29336,6 @@ static int findCreateFileMode( } }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ *pMode = 0600; - }else{ - *pMode = SQLITE_DEFAULT_FILE_PERMISSIONS; } return rc; } @@ -31162,6 +31543,7 @@ struct winFile { #endif }; + /* ** Forward prototypes. */ @@ -31329,7 +31711,7 @@ SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ ** Convert UTF-8 to multibyte character string. Space to hold the ** returned string is obtained from malloc(). */ -static char *utf8ToMbcs(const char *zFilename){ +SQLITE_API char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ char *zFilenameMbcs; WCHAR *zTmpWide; @@ -31342,6 +31724,109 @@ static char *utf8ToMbcs(const char *zFilename){ return zFilenameMbcs; } + +/* +** The return value of getLastErrorMsg +** is zero if the error message fits in the buffer, or non-zero +** otherwise (if the message was truncated). +*/ +static int getLastErrorMsg(int nBuf, char *zBuf){ + /* FormatMessage returns 0 on failure. Otherwise it + ** returns the number of TCHARs written to the output + ** buffer, excluding the terminating null char. + */ + DWORD error = GetLastError(); + DWORD dwLen = 0; + char *zOut = 0; + + if( isNT() ){ + WCHAR *zTempWide = NULL; + dwLen = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + error, + 0, + (LPWSTR) &zTempWide, + 0, + 0); + if( dwLen > 0 ){ + /* allocate a buffer and convert to UTF8 */ + zOut = unicodeToUtf8(zTempWide); + /* free the system buffer allocated by FormatMessage */ + LocalFree(zTempWide); + } +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. +*/ +#if SQLITE_OS_WINCE==0 + }else{ + char *zTemp = NULL; + dwLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + error, + 0, + (LPSTR) &zTemp, + 0, + 0); + if( dwLen > 0 ){ + /* allocate a buffer and convert to UTF8 */ + zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + /* free the system buffer allocated by FormatMessage */ + LocalFree(zTemp); + } +#endif + } + if( 0 == dwLen ){ + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", error, error); + }else{ + /* copy a maximum of nBuf chars to output buffer */ + sqlite3_snprintf(nBuf, zBuf, "%s", zOut); + /* free the UTF8 buffer */ + free(zOut); + } + return 0; +} + +/* +** +** This function - winLogErrorAtLine() - is only ever called via the macro +** winLogError(). +** +** This routine is invoked after an error occurs in an OS function. +** It logs a message using sqlite3_log() containing the current value of +** error code and, if possible, the human-readable equivalent from +** FormatMessage. +** +** The first argument passed to the macro should be the error code that +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** The two subsequent arguments should be the name of the OS function that +** failed and the the associated file-system path, if any. +*/ +#define winLogError(a,b,c) winLogErrorAtLine(a,b,c,__LINE__) +static int winLogErrorAtLine( + int errcode, /* SQLite error code */ + const char *zFunc, /* Name of OS function that failed */ + const char *zPath, /* File path associated with error */ + int iLine /* Source line number where error occurred */ +){ + char zMsg[500]; /* Human readable error text */ + int i; /* Loop counter */ + DWORD iErrno = GetLastError(); /* Error code */ + + zMsg[0] = 0; + getLastErrorMsg(sizeof(zMsg), zMsg); + assert( errcode!=SQLITE_OK ); + if( zPath==0 ) zPath = ""; + for(i=0; zMsg[i] && zMsg[i]!='\r' && zMsg[i]!='\n'; i++){} + zMsg[i] = 0; + sqlite3_log(errcode, + "os_win.c:%d: (%d) %s(%s) - %s", + iLine, iErrno, zFunc, zPath, zMsg + ); + + return errcode; +} + #if SQLITE_OS_WINCE /************************************************************************* ** This section contains code for WinCE only. @@ -31418,6 +31903,7 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ pFile->hMutex = CreateMutexW(NULL, FALSE, zName); if (!pFile->hMutex){ pFile->lastErrno = GetLastError(); + winLogError(SQLITE_ERROR, "winceCreateLock1", zFilename); free(zName); return FALSE; } @@ -31449,6 +31935,7 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ /* If mapping failed, close the shared memory handle and erase it */ if (!pFile->shared){ pFile->lastErrno = GetLastError(); + winLogError(SQLITE_ERROR, "winceCreateLock2", zFilename); CloseHandle(pFile->hShared); pFile->hShared = NULL; } @@ -31694,6 +32181,7 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ dwRet = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); if( (dwRet==INVALID_SET_FILE_POINTER && GetLastError()!=NO_ERROR) ){ pFile->lastErrno = GetLastError(); + winLogError(SQLITE_IOERR_SEEK, "seekWinFile", pFile->zPath); return 1; } @@ -31739,7 +32227,8 @@ static int winClose(sqlite3_file *id){ #endif OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); OpenCounter(-1); - return rc ? SQLITE_OK : SQLITE_IOERR; + return rc ? SQLITE_OK + : winLogError(SQLITE_IOERR_CLOSE, "winClose", pFile->zPath); } /* @@ -31765,7 +32254,7 @@ static int winRead( } if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ pFile->lastErrno = GetLastError(); - return SQLITE_IOERR_READ; + return winLogError(SQLITE_IOERR_READ, "winRead", pFile->zPath); } if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ @@ -31813,10 +32302,11 @@ static int winWrite( } if( rc ){ - if( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ){ + if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) + || ( pFile->lastErrno==ERROR_DISK_FULL )){ return SQLITE_FULL; } - return SQLITE_IOERR_WRITE; + return winLogError(SQLITE_IOERR_WRITE, "winWrite", pFile->zPath); } return SQLITE_OK; } @@ -31844,10 +32334,10 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( seekWinFile(pFile, nByte) ){ - rc = SQLITE_IOERR_TRUNCATE; + rc = winLogError(SQLITE_IOERR_TRUNCATE, "winTruncate1", pFile->zPath); }else if( 0==SetEndOfFile(pFile->h) ){ pFile->lastErrno = GetLastError(); - rc = SQLITE_IOERR_TRUNCATE; + rc = winLogError(SQLITE_IOERR_TRUNCATE, "winTruncate2", pFile->zPath); } OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); @@ -31869,6 +32359,7 @@ SQLITE_API int sqlite3_fullsync_count = 0; static int winSync(sqlite3_file *id, int flags){ #if !defined(NDEBUG) || !defined(SQLITE_NO_SYNC) || defined(SQLITE_DEBUG) winFile *pFile = (winFile*)id; + BOOL rc; #else UNUSED_PARAMETER(id); #endif @@ -31881,20 +32372,19 @@ static int winSync(sqlite3_file *id, int flags){ OSTRACE(("SYNC %d lock=%d\n", pFile->h, pFile->locktype)); -#ifndef SQLITE_TEST - UNUSED_PARAMETER(flags); -#else - if( flags & SQLITE_SYNC_FULL ){ - sqlite3_fullsync_count++; - } - sqlite3_sync_count++; -#endif - /* Unix cannot, but some systems may return SQLITE_FULL from here. This ** line is to test that doing so does not cause any problems. */ SimulateDiskfullError( return SQLITE_FULL ); - SimulateIOError( return SQLITE_IOERR; ); + +#ifndef SQLITE_TEST + UNUSED_PARAMETER(flags); +#else + if( (flags&0x0F)==SQLITE_SYNC_FULL ){ + sqlite3_fullsync_count++; + } + sqlite3_sync_count++; +#endif /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a ** no-op @@ -31902,11 +32392,13 @@ static int winSync(sqlite3_file *id, int flags){ #ifdef SQLITE_NO_SYNC return SQLITE_OK; #else - if( FlushFileBuffers(pFile->h) ){ + rc = FlushFileBuffers(pFile->h); + SimulateIOError( rc=FALSE ); + if( rc ){ return SQLITE_OK; }else{ pFile->lastErrno = GetLastError(); - return SQLITE_IOERR; + return winLogError(SQLITE_IOERR_FSYNC, "winSync", pFile->zPath); } #endif } @@ -31927,7 +32419,7 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ && ((error = GetLastError()) != NO_ERROR) ) { pFile->lastErrno = error; - return SQLITE_IOERR_FSTAT; + return winLogError(SQLITE_IOERR_FSTAT, "winFileSize", pFile->zPath); } *pSize = (((sqlite3_int64)upperBits)<<32) + lowerBits; return SQLITE_OK; @@ -31966,6 +32458,7 @@ static int getReadLock(winFile *pFile){ } if( res == 0 ){ pFile->lastErrno = GetLastError(); + /* No need to log a failure to lock */ } return res; } @@ -31984,8 +32477,9 @@ static int unlockReadLock(winFile *pFile){ res = UnlockFile(pFile->h, SHARED_FIRST + pFile->sharedLockByte, 0, 1, 0); #endif } - if( res == 0 ){ + if( res==0 && GetLastError()!=ERROR_NOT_LOCKED ){ pFile->lastErrno = GetLastError(); + winLogError(SQLITE_IOERR_UNLOCK, "unlockReadLock", pFile->zPath); } return res; } @@ -32186,7 +32680,7 @@ static int winUnlock(sqlite3_file *id, int locktype){ if( locktype==SHARED_LOCK && !getReadLock(pFile) ){ /* This should never happen. We should always be able to ** reacquire the read lock */ - rc = SQLITE_IOERR_UNLOCK; + rc = winLogError(SQLITE_IOERR_UNLOCK, "winUnlock", pFile->zPath); } } if( type>=RESERVED_LOCK ){ @@ -32501,6 +32995,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ memset(pNew, 0, sizeof(*pNew)); pNew->zFilename = (char*)&pNew[1]; sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); + sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); /* Look to see if there is an existing winShmNode that can be used. ** If no matching winShmNode currently exists, create a new one. @@ -32543,7 +33038,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ if( winShmSystemLock(pShmNode, _SHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); if( rc!=SQLITE_OK ){ - rc = SQLITE_IOERR_SHMOPEN; + rc = winLogError(SQLITE_IOERR_SHMOPEN, "winOpenShm", pDbFd->zPath); } } if( rc==SQLITE_OK ){ @@ -32802,7 +33297,7 @@ static int winShmMap( */ rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); if( rc!=SQLITE_OK ){ - rc = SQLITE_IOERR_SHMSIZE; + rc = winLogError(SQLITE_IOERR_SHMSIZE, "winShmMap1", pDbFd->zPath); goto shmpage_out; } @@ -32816,7 +33311,7 @@ static int winShmMap( if( !isWrite ) goto shmpage_out; rc = winTruncate((sqlite3_file *)&pShmNode->hFile, nByte); if( rc!=SQLITE_OK ){ - rc = SQLITE_IOERR_SHMSIZE; + rc = winLogError(SQLITE_IOERR_SHMSIZE, "winShmMap2", pDbFd->zPath); goto shmpage_out; } } @@ -32853,7 +33348,7 @@ static int winShmMap( } if( !pMap ){ pShmNode->lastErrno = GetLastError(); - rc = SQLITE_IOERR; + rc = winLogError(SQLITE_IOERR_SHMMAP, "winShmMap3", pDbFd->zPath); if( hMap ) CloseHandle(hMap); goto shmpage_out; } @@ -32935,7 +33430,7 @@ static void *convertUtf8Filename(const char *zFilename){ */ #if SQLITE_OS_WINCE==0 }else{ - zConverted = utf8ToMbcs(zFilename); + zConverted = sqlite3_win32_utf8_to_mbcs(zFilename); #endif } /* caller will handle out of memory */ @@ -33015,68 +33510,6 @@ static int getTempname(int nBuf, char *zBuf){ return SQLITE_OK; } -/* -** The return value of getLastErrorMsg -** is zero if the error message fits in the buffer, or non-zero -** otherwise (if the message was truncated). -*/ -static int getLastErrorMsg(int nBuf, char *zBuf){ - /* FormatMessage returns 0 on failure. Otherwise it - ** returns the number of TCHARs written to the output - ** buffer, excluding the terminating null char. - */ - DWORD error = GetLastError(); - DWORD dwLen = 0; - char *zOut = 0; - - if( isNT() ){ - WCHAR *zTempWide = NULL; - dwLen = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - error, - 0, - (LPWSTR) &zTempWide, - 0, - 0); - if( dwLen > 0 ){ - /* allocate a buffer and convert to UTF8 */ - zOut = unicodeToUtf8(zTempWide); - /* free the system buffer allocated by FormatMessage */ - LocalFree(zTempWide); - } -/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, -** it's important to not reference them for WINCE builds. -*/ -#if SQLITE_OS_WINCE==0 - }else{ - char *zTemp = NULL; - dwLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - error, - 0, - (LPSTR) &zTemp, - 0, - 0); - if( dwLen > 0 ){ - /* allocate a buffer and convert to UTF8 */ - zOut = sqlite3_win32_mbcs_to_utf8(zTemp); - /* free the system buffer allocated by FormatMessage */ - LocalFree(zTemp); - } -#endif - } - if( 0 == dwLen ){ - sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", error, error); - }else{ - /* copy a maximum of nBuf chars to output buffer */ - sqlite3_snprintf(nBuf, zBuf, "%s", zOut); - /* free the UTF8 buffer */ - free(zOut); - } - return 0; -} - /* ** Open a file. */ @@ -33248,6 +33681,7 @@ static int winOpen( if( h==INVALID_HANDLE_VALUE ){ pFile->lastErrno = GetLastError(); + winLogError(SQLITE_CANTOPEN, "winOpen", zUtf8Name); free(zConverted); if( isReadWrite ){ return winOpen(pVfs, zName, id, @@ -33351,7 +33785,8 @@ static int winDelete( "ok" : "failed" )); return ( (rc == INVALID_FILE_ATTRIBUTES) - && (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK : SQLITE_IOERR_DELETE; + && (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK : + winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename); } /* @@ -33391,6 +33826,7 @@ static int winAccess( } }else{ if( GetLastError()!=ERROR_FILE_NOT_FOUND ){ + winLogError(SQLITE_IOERR_ACCESS, "winAccess", zFilename); free(zConverted); return SQLITE_IOERR_ACCESS; }else{ @@ -33455,6 +33891,13 @@ static int winFullPathname( void *zConverted; char *zOut; + /* If this path name begins with "/X:", where "X" is any alphabetic + ** character, discard the initial "/" from the pathname. + */ + if( zRelative[0]=='/' && sqlite3Isalpha(zRelative[1]) && zRelative[2]==':' ){ + zRelative++; + } + /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this ** function failing. This function could fail if, for example, the @@ -34495,6 +34938,13 @@ SQLITE_PRIVATE int sqlite3PcacheFetch( } if( pPg ){ int rc; +#ifdef SQLITE_LOG_CACHE_SPILL + sqlite3_log(SQLITE_FULL, + "spill page %d making room for %d - cache used: %d/%d", + pPg->pgno, pgno, + sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), + pCache->nMax); +#endif rc = pCache->xStress(pCache->pStress, pPg); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ return rc; @@ -35405,7 +35855,7 @@ static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){ pGroup = (PGroup*)&pCache[1]; pGroup->mxPinned = 10; }else{ - pGroup = &pcache1_g.grp; + pGroup = &pcache1.grp; } pCache->pGroup = pGroup; pCache->szPage = szPage; @@ -36266,6 +36716,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, u8 iBatch, sqlite3_int64 i #ifdef SQLITE_OMIT_WAL # define sqlite3WalOpen(x,y,z) 0 +# define sqlite3WalLimit(x,y) # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) @@ -36291,9 +36742,12 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, u8 iBatch, sqlite3_int64 i typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -SQLITE_PRIVATE int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *zName, int, Wal**); +SQLITE_PRIVATE int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); SQLITE_PRIVATE int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); +/* Set the limiting size of a WAL file. */ +SQLITE_PRIVATE void sqlite3WalLimit(Wal*, i64); + /* Used by readers to open (lock) and close (unlock) a snapshot. A ** snapshot is like a read-transaction. It is the state of the database ** at an instant in time. sqlite3WalOpenSnapshot gets a read lock and @@ -40642,6 +41096,8 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int noReadlock = (flags & PAGER_NO_READLOCK)!=0; /* True to omit read-lock */ int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ + const char *zUri = 0; /* URI args to copy */ + int nUri = 0; /* Number of bytes of URI args at *zUri */ /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). This @@ -40672,6 +41128,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( ** leave both nPathname and zPathname set to 0. */ if( zFilename && zFilename[0] ){ + const char *z; nPathname = pVfs->mxPathname+1; zPathname = sqlite3Malloc(nPathname*2); if( zPathname==0 ){ @@ -40680,6 +41137,12 @@ SQLITE_PRIVATE int sqlite3PagerOpen( zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); nPathname = sqlite3Strlen30(zPathname); + z = zUri = &zFilename[sqlite3Strlen30(zFilename)+1]; + while( *z ){ + z += sqlite3Strlen30(z)+1; + z += sqlite3Strlen30(z)+1; + } + nUri = &z[1] - zUri; if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ /* This branch is taken when the journal path required by ** the database being opened will be more than pVfs->mxPathname @@ -40712,7 +41175,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( ROUND8(pcacheSize) + /* PCache object */ ROUND8(pVfs->szOsFile) + /* The main db file */ journalFileSize * 2 + /* The two journal files */ - nPathname + 1 + /* zFilename */ + nPathname + 1 + nUri + /* zFilename */ nPathname + 8 + 1 /* zJournal */ #ifndef SQLITE_OMIT_WAL + nPathname + 4 + 1 /* zWal */ @@ -40734,14 +41197,17 @@ SQLITE_PRIVATE int sqlite3PagerOpen( /* Fill in the Pager.zFilename and Pager.zJournal buffers, if required. */ if( zPathname ){ assert( nPathname>0 ); - pPager->zJournal = (char*)(pPtr += nPathname + 1); + pPager->zJournal = (char*)(pPtr += nPathname + 1 + nUri); memcpy(pPager->zFilename, zPathname, nPathname); + memcpy(&pPager->zFilename[nPathname+1], zUri, nUri); memcpy(pPager->zJournal, zPathname, nPathname); memcpy(&pPager->zJournal[nPathname], "-journal", 8); + sqlite3FileSuffix3(pPager->zFilename, pPager->zJournal); #ifndef SQLITE_OMIT_WAL pPager->zWal = &pPager->zJournal[nPathname+8+1]; memcpy(pPager->zWal, zPathname, nPathname); memcpy(&pPager->zWal[nPathname], "-wal", 4); + sqlite3FileSuffix3(pPager->zFilename, pPager->zWal); #endif sqlite3_free(zPathname); } @@ -42086,8 +42552,8 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( pList = pPageOne; pList->pDirty = 0; } - assert( pList!=0 || rc!=SQLITE_OK ); - if( pList ){ + assert( rc==SQLITE_OK ); + if( ALWAYS(pList) ){ rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1, (pPager->fullSync ? pPager->syncFlags : 0) ); @@ -42950,6 +43416,7 @@ SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager *pPager){ SQLITE_PRIVATE i64 sqlite3PagerJournalSizeLimit(Pager *pPager, i64 iLimit){ if( iLimit>=-1 ){ pPager->journalSizeLimit = iLimit; + sqlite3WalLimit(pPager->pWal, iLimit); } return pPager->journalSizeLimit; } @@ -43041,7 +43508,8 @@ static int pagerOpenWal(Pager *pPager){ */ if( rc==SQLITE_OK ){ rc = sqlite3WalOpen(pPager->pVfs, - pPager->fd, pPager->zWal, pPager->exclusiveMode, &pPager->pWal + pPager->fd, pPager->zWal, pPager->exclusiveMode, + pPager->journalSizeLimit, &pPager->pWal ); } @@ -43573,6 +44041,7 @@ struct Wal { sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ u32 iCallback; /* Value to pass to log callback (or 0) */ + i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ volatile u32 **apWiData; /* Pointer to wal-index content in memory */ u32 szPage; /* Database page size */ @@ -43580,7 +44049,7 @@ struct Wal { u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ u8 writeLock; /* True if in a write transaction */ u8 ckptLock; /* True if holding a checkpoint lock */ - u8 readOnly; /* True if the WAL file is open read-only */ + u8 readOnly; /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */ WalIndexHdr hdr; /* Wal-index header for current transaction */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ @@ -43596,6 +44065,13 @@ struct Wal { #define WAL_EXCLUSIVE_MODE 1 #define WAL_HEAPMEMORY_MODE 2 +/* +** Possible values for WAL.readOnly +*/ +#define WAL_RDWR 0 /* Normal read/write connection */ +#define WAL_RDONLY 1 /* The WAL file is readonly */ +#define WAL_SHM_RDONLY 2 /* The SHM file is readonly */ + /* ** Each page of the wal-index mapping contains a hash-table made up of ** an array of HASHTABLE_NSLOT elements of the following type. @@ -43689,6 +44165,10 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); + if( rc==SQLITE_READONLY ){ + pWal->readOnly |= WAL_SHM_RDONLY; + rc = SQLITE_OK; + } } } @@ -44395,6 +44875,7 @@ SQLITE_PRIVATE int sqlite3WalOpen( sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ + i64 mxWalSize, /* Truncate WAL to this size on reset */ Wal **ppWal /* OUT: Allocated Wal handle */ ){ int rc; /* Return Code */ @@ -44427,6 +44908,7 @@ SQLITE_PRIVATE int sqlite3WalOpen( pRet->pWalFd = (sqlite3_file *)&pRet[1]; pRet->pDbFd = pDbFd; pRet->readLock = -1; + pRet->mxWalSize = mxWalSize; pRet->zWalName = zWalName; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); @@ -44434,7 +44916,7 @@ SQLITE_PRIVATE int sqlite3WalOpen( flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ - pRet->readOnly = 1; + pRet->readOnly = WAL_RDONLY; } if( rc!=SQLITE_OK ){ @@ -44448,6 +44930,13 @@ SQLITE_PRIVATE int sqlite3WalOpen( return rc; } +/* +** Change the size to which the WAL file is trucated on each reset. +*/ +SQLITE_PRIVATE void sqlite3WalLimit(Wal *pWal, i64 iLimit){ + if( pWal ) pWal->mxWalSize = iLimit; +} + /* ** Find the smallest page number out of all pages held in the WAL that ** has not been returned by any prior invocation of this method on the @@ -45068,21 +45557,28 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** with a writer. So get a WRITE lock and try again. */ assert( badHdr==0 || pWal->writeLock==0 ); - if( badHdr && SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ - pWal->writeLock = 1; - if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ - badHdr = walIndexTryHdr(pWal, pChanged); - if( badHdr ){ - /* If the wal-index header is still malformed even while holding - ** a WRITE lock, it can only mean that the header is corrupted and - ** needs to be reconstructed. So run recovery to do exactly that. - */ - rc = walIndexRecover(pWal); - *pChanged = 1; + if( badHdr ){ + if( pWal->readOnly & WAL_SHM_RDONLY ){ + if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ + walUnlockShared(pWal, WAL_WRITE_LOCK); + rc = SQLITE_READONLY_RECOVERY; } + }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ + pWal->writeLock = 1; + if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ + badHdr = walIndexTryHdr(pWal, pChanged); + if( badHdr ){ + /* If the wal-index header is still malformed even while holding + ** a WRITE lock, it can only mean that the header is corrupted and + ** needs to be reconstructed. So run recovery to do exactly that. + */ + rc = walIndexRecover(pWal); + *pChanged = 1; + } + } + pWal->writeLock = 0; + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } - pWal->writeLock = 0; - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } /* If the header is read successfully, check the version number to make @@ -45269,7 +45765,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } /* There was once an "if" here. The extra "{" is to preserve indentation. */ { - if( mxReadMark < pWal->hdr.mxFrame || mxI==0 ){ + if( (pWal->readOnly & WAL_SHM_RDONLY)==0 + && (mxReadMarkhdr.mxFrame || mxI==0) + ){ for(i=1; ireadOnly & WAL_SHM_RDONLY)!=0 ); + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; } rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); @@ -45683,6 +46181,24 @@ static int walRestartLog(Wal *pWal){ */ int i; /* Loop counter */ u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ + + /* Limit the size of WAL file if the journal_size_limit PRAGMA is + ** set to a non-negative value. Log errors encountered + ** during the truncation attempt. */ + if( pWal->mxWalSize>=0 ){ + i64 sz; + int rx; + sqlite3BeginBenignMalloc(); + rx = sqlite3OsFileSize(pWal->pWalFd, &sz); + if( rx==SQLITE_OK && (sz > pWal->mxWalSize) ){ + rx = sqlite3OsTruncate(pWal->pWalFd, pWal->mxWalSize); + } + sqlite3EndBenignMalloc(); + if( rx ){ + sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName); + } + } + pWal->nCkpt++; pWal->hdr.mxFrame = 0; sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); @@ -45908,6 +46424,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( assert( pWal->ckptLock==0 ); assert( pWal->writeLock==0 ); + if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc ){ @@ -47788,6 +48305,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ *pRC = SQLITE_CORRUPT_BKPT; goto ptrmap_exit; } + assert( offset <= (int)pBt->usableSize-5 ); pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ @@ -47827,6 +48345,11 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ + sqlite3PagerUnref(pDbPage); + return SQLITE_CORRUPT_BKPT; + } + assert( offset <= (int)pBt->usableSize-5 ); assert( pEType!=0 ); *pEType = pPtrmap[offset]; if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]); @@ -47851,6 +48374,8 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ */ #define findCell(P,I) \ ((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)]))) +#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I))))) + /* ** This a more complex version of findCell() that works for @@ -48688,13 +49213,13 @@ static int btreeInvokeBusyHandler(void *pArg){ ** to problems with locking. */ SQLITE_PRIVATE int sqlite3BtreeOpen( + sqlite3_vfs *pVfs, /* VFS to use for this b-tree */ const char *zFilename, /* Name of the file containing the BTree database */ sqlite3 *db, /* Associated database handle */ Btree **ppBtree, /* Pointer to new Btree object written here */ int flags, /* Options */ int vfsFlags /* Flags passed through to sqlite3_vfs.xOpen() */ ){ - sqlite3_vfs *pVfs; /* The VFS to use for this btree */ BtShared *pBt = 0; /* Shared part of btree structure */ Btree *p; /* Handle to return */ sqlite3_mutex *mutexOpen = 0; /* Prevents a race condition. Ticket #3537 */ @@ -48716,6 +49241,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( #endif assert( db!=0 ); + assert( pVfs!=0 ); assert( sqlite3_mutex_held(db->mutex) ); assert( (flags&0xff)==flags ); /* flags fit in 8 bits */ @@ -48734,7 +49260,6 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; } - pVfs = db->pVfs; p = sqlite3MallocZero(sizeof(Btree)); if( !p ){ return SQLITE_NOMEM; @@ -51445,7 +51970,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( } assert( pCur->apPage[0]->intKey || pIdxKey ); for(;;){ - int lwr, upr; + int lwr, upr, idx; Pgno chldPg; MemPage *pPage = pCur->apPage[pCur->iPage]; int c; @@ -51461,14 +51986,14 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( lwr = 0; upr = pPage->nCell-1; if( biasRight ){ - pCur->aiIdx[pCur->iPage] = (u16)upr; + pCur->aiIdx[pCur->iPage] = (u16)(idx = upr); }else{ - pCur->aiIdx[pCur->iPage] = (u16)((upr+lwr)/2); + pCur->aiIdx[pCur->iPage] = (u16)(idx = (upr+lwr)/2); } for(;;){ - int idx = pCur->aiIdx[pCur->iPage]; /* Index of current cell in pPage */ u8 *pCell; /* Pointer to current cell in pPage */ + assert( idx==pCur->aiIdx[pCur->iPage] ); pCur->info.nSize = 0; pCell = findCell(pPage, idx) + pPage->childPtrSize; if( pPage->intKey ){ @@ -51551,7 +52076,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( if( lwr>upr ){ break; } - pCur->aiIdx[pCur->iPage] = (u16)((lwr+upr)/2); + pCur->aiIdx[pCur->iPage] = (u16)(idx = (lwr+upr)/2); } assert( lwr==upr+1 ); assert( pPage->isInit ); @@ -52384,10 +52909,10 @@ static int fillInCell( ** "sz" must be the number of bytes in the cell. */ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ - int i; /* Loop counter */ u32 pc; /* Offset to cell content of cell being deleted */ u8 *data; /* pPage->aData */ u8 *ptr; /* Used to move bytes around within data[] */ + u8 *endPtr; /* End of loop */ int rc; /* The return code */ int hdr; /* Beginning of the header. 0 most pages. 100 page 1 */ @@ -52412,9 +52937,11 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ *pRC = rc; return; } - for(i=idx+1; inCell; i++, ptr+=2){ - ptr[0] = ptr[2]; - ptr[1] = ptr[3]; + endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2]; + assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */ + while( ptrnCell--; put2byte(&data[hdr+3], pPage->nCell); @@ -52454,6 +52981,7 @@ static void insertCell( int cellOffset; /* Address of first cell pointer in data[] */ u8 *data; /* The content of the whole page */ u8 *ptr; /* Used for moving information around in data[] */ + u8 *endPtr; /* End of the loop */ int nSkip = (iChild ? 4 : 0); @@ -52504,9 +53032,12 @@ static void insertCell( if( iChild ){ put4byte(&data[idx], iChild); } - for(j=end, ptr=&data[j]; j>ins; j-=2, ptr-=2){ - ptr[0] = ptr[-2]; - ptr[1] = ptr[-1]; + ptr = &data[end]; + endPtr = &data[ins]; + assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */ + while( ptr>endPtr ){ + *(u16*)ptr = *(u16*)&ptr[-2]; + ptr -= 2; } put2byte(&data[ins], idx); put2byte(&data[pPage->hdrOffset+3], pPage->nCell); @@ -52551,10 +53082,11 @@ static void assemblePage( pCellptr = &data[pPage->cellOffset + nCell*2]; cellbody = nUsable; for(i=nCell-1; i>=0; i--){ + u16 sz = aSize[i]; pCellptr -= 2; - cellbody -= aSize[i]; + cellbody -= sz; put2byte(pCellptr, cellbody); - memcpy(&data[cellbody], apCell[i], aSize[i]); + memcpy(&data[cellbody], apCell[i], sz); } put2byte(&data[hdr+3], nCell); put2byte(&data[hdr+5], cellbody); @@ -53008,12 +53540,24 @@ static int balance_nonroot( memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); limit = pOld->nCell+pOld->nOverflow; - for(j=0; jnOverflow>0 ){ + for(j=0; jaData; + u16 maskPage = pOld->maskPage; + u16 cellOffset = pOld->cellOffset; + for(j=0; jp3 = p3; pOp->p4.p = 0; pOp->p4type = P4_NOTUSED; - p->expired = 0; - if( op==OP_ParseSchema ){ - /* Any program that uses the OP_ParseSchema opcode needs to lock - ** all btrees. */ - int j; - for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); - } #ifdef SQLITE_DEBUG pOp->zComment = 0; if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]); @@ -57207,6 +57744,20 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4( return addr; } +/* +** Add an OP_ParseSchema opcode. This routine is broken out from +** sqlite3VdbeAddOp4() since it needs to also local all btrees. +** +** The zWhere string must have been obtained from sqlite3_malloc(). +** This routine will take ownership of the allocated memory. +*/ +SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ + int j; + int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0); + sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC); + for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); +} + /* ** Add an opcode that includes the p4 value as an integer. */ @@ -58397,34 +58948,13 @@ static void *allocSpace( } /* -** Prepare a virtual machine for execution. This involves things such -** as allocating stack space and initializing the program counter. -** After the VDBE has be prepped, it can be executed by one or more -** calls to sqlite3VdbeExec(). -** -** This is the only way to move a VDBE from VDBE_MAGIC_INIT to -** VDBE_MAGIC_RUN. -** -** This function may be called more than once on a single virtual machine. -** The first call is made while compiling the SQL statement. Subsequent -** calls are made as part of the process of resetting a statement to be -** re-executed (from a call to sqlite3_reset()). The nVar, nMem, nCursor -** and isExplain parameters are only passed correct values the first time -** the function is called. On subsequent calls, from sqlite3_reset(), nVar -** is passed -1 and nMem, nCursor and isExplain are all passed zero. +** Rewind the VDBE back to the beginning in preparation for +** running it. */ -SQLITE_PRIVATE void sqlite3VdbeMakeReady( - Vdbe *p, /* The VDBE */ - int nVar, /* Number of '?' see in the SQL statement */ - int nMem, /* Number of memory cells to allocate */ - int nCursor, /* Number of cursors to allocate */ - int nArg, /* Maximum number of args in SubPrograms */ - int isExplain, /* True if the EXPLAIN keywords is present */ - int usesStmtJournal /* True to set Vdbe.usesStmtJournal */ -){ - int n; - sqlite3 *db = p->db; - +SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + int i; +#endif assert( p!=0 ); assert( p->magic==VDBE_MAGIC_INIT ); @@ -58435,6 +58965,71 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( /* Set the magic to VDBE_MAGIC_RUN sooner rather than later. */ p->magic = VDBE_MAGIC_RUN; +#ifdef SQLITE_DEBUG + for(i=1; inMem; i++){ + assert( p->aMem[i].db==p->db ); + } +#endif + p->pc = -1; + p->rc = SQLITE_OK; + p->errorAction = OE_Abort; + p->magic = VDBE_MAGIC_RUN; + p->nChange = 0; + p->cacheCtr = 1; + p->minWriteFileFormat = 255; + p->iStatement = 0; + p->nFkConstraint = 0; +#ifdef VDBE_PROFILE + for(i=0; inOp; i++){ + p->aOp[i].cnt = 0; + p->aOp[i].cycles = 0; + } +#endif +} + +/* +** Prepare a virtual machine for execution for the first time after +** creating the virtual machine. This involves things such +** as allocating stack space and initializing the program counter. +** After the VDBE has be prepped, it can be executed by one or more +** calls to sqlite3VdbeExec(). +** +** This function may be called exact once on a each virtual machine. +** After this routine is called the VM has been "packaged" and is ready +** to run. After this routine is called, futher calls to +** sqlite3VdbeAddOp() functions are prohibited. This routine disconnects +** the Vdbe from the Parse object that helped generate it so that the +** the Vdbe becomes an independent entity and the Parse object can be +** destroyed. +** +** Use the sqlite3VdbeRewind() procedure to restore a virtual machine back +** to its initial state after it has been run. +*/ +SQLITE_PRIVATE void sqlite3VdbeMakeReady( + Vdbe *p, /* The VDBE */ + Parse *pParse /* Parsing context */ +){ + sqlite3 *db; /* The database connection */ + int nVar; /* Number of parameters */ + int nMem; /* Number of VM memory registers */ + int nCursor; /* Number of cursors required */ + int nArg; /* Number of arguments in subprograms */ + int n; /* Loop counter */ + u8 *zCsr; /* Memory available for allocation */ + u8 *zEnd; /* First byte past allocated memory */ + int nByte; /* How much extra memory is needed */ + + assert( p!=0 ); + assert( p->nOp>0 ); + assert( pParse!=0 ); + assert( p->magic==VDBE_MAGIC_INIT ); + db = p->db; + assert( db->mallocFailed==0 ); + nVar = pParse->nVar; + nMem = pParse->nMem; + nCursor = pParse->nTab; + nArg = pParse->nMaxArg; + /* For each cursor required, also allocate a memory cell. Memory ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by ** the vdbe program. Instead they are used to allocate space for @@ -58447,91 +59042,69 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( nMem += nCursor; /* Allocate space for memory registers, SQL variables, VDBE cursors and - ** an array to marshal SQL function arguments in. This is only done the - ** first time this function is called for a given VDBE, not when it is - ** being called from sqlite3_reset() to reset the virtual machine. + ** an array to marshal SQL function arguments in. */ - if( nVar>=0 && ALWAYS(db->mallocFailed==0) ){ - u8 *zCsr = (u8 *)&p->aOp[p->nOp]; /* Memory avaliable for alloation */ - u8 *zEnd = (u8 *)&p->aOp[p->nOpAlloc]; /* First byte past available mem */ - int nByte; /* How much extra memory needed */ + zCsr = (u8*)&p->aOp[p->nOp]; /* Memory avaliable for allocation */ + zEnd = (u8*)&p->aOp[p->nOpAlloc]; /* First byte past end of zCsr[] */ - resolveP2Values(p, &nArg); - p->usesStmtJournal = (u8)usesStmtJournal; - if( isExplain && nMem<10 ){ - nMem = 10; + resolveP2Values(p, &nArg); + p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); + if( pParse->explain && nMem<10 ){ + nMem = 10; + } + memset(zCsr, 0, zEnd-zCsr); + zCsr += (zCsr - (u8*)0)&7; + assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + p->expired = 0; + + /* Memory for registers, parameters, cursor, etc, is allocated in two + ** passes. On the first pass, we try to reuse unused space at the + ** end of the opcode array. If we are unable to satisfy all memory + ** requirements by reusing the opcode array tail, then the second + ** pass will fill in the rest using a fresh allocation. + ** + ** This two-pass approach that reuses as much memory as possible from + ** the leftover space at the end of the opcode array can significantly + ** reduce the amount of memory held by a prepared statement. + */ + do { + nByte = 0; + p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); + p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); + p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); + p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); + p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), + &zCsr, zEnd, &nByte); + if( nByte ){ + p->pFree = sqlite3DbMallocZero(db, nByte); } - memset(zCsr, 0, zEnd-zCsr); - zCsr += (zCsr - (u8*)0)&7; - assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + zCsr = p->pFree; + zEnd = &zCsr[nByte]; + }while( nByte && !db->mallocFailed ); - /* Memory for registers, parameters, cursor, etc, is allocated in two - ** passes. On the first pass, we try to reuse unused space at the - ** end of the opcode array. If we are unable to satisfy all memory - ** requirements by reusing the opcode array tail, then the second - ** pass will fill in the rest using a fresh allocation. - ** - ** This two-pass approach that reuses as much memory as possible from - ** the leftover space at the end of the opcode array can significantly - ** reduce the amount of memory held by a prepared statement. - */ - do { - nByte = 0; - p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); - p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); - p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); - p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); - p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), - &zCsr, zEnd, &nByte); - if( nByte ){ - p->pFree = sqlite3DbMallocZero(db, nByte); - } - zCsr = p->pFree; - zEnd = &zCsr[nByte]; - }while( nByte && !db->mallocFailed ); - - p->nCursor = (u16)nCursor; - if( p->aVar ){ - p->nVar = (ynVar)nVar; - for(n=0; naVar[n].flags = MEM_Null; - p->aVar[n].db = db; - } - } - if( p->aMem ){ - p->aMem--; /* aMem[] goes from 1..nMem */ - p->nMem = nMem; /* not from 0..nMem-1 */ - for(n=1; n<=nMem; n++){ - p->aMem[n].flags = MEM_Null; - p->aMem[n].db = db; - } + p->nCursor = (u16)nCursor; + if( p->aVar ){ + p->nVar = (ynVar)nVar; + for(n=0; naVar[n].flags = MEM_Null; + p->aVar[n].db = db; } } -#ifdef SQLITE_DEBUG - for(n=1; nnMem; n++){ - assert( p->aMem[n].db==db ); + if( p->azVar ){ + p->nzVar = pParse->nzVar; + memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); + memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); } -#endif - - p->pc = -1; - p->rc = SQLITE_OK; - p->errorAction = OE_Abort; - p->explain |= isExplain; - p->magic = VDBE_MAGIC_RUN; - p->nChange = 0; - p->cacheCtr = 1; - p->minWriteFileFormat = 255; - p->iStatement = 0; - p->nFkConstraint = 0; -#ifdef VDBE_PROFILE - { - int i; - for(i=0; inOp; i++){ - p->aOp[i].cnt = 0; - p->aOp[i].cycles = 0; + if( p->aMem ){ + p->aMem--; /* aMem[] goes from 1..nMem */ + p->nMem = nMem; /* not from 0..nMem-1 */ + for(n=1; n<=nMem; n++){ + p->aMem[n].flags = MEM_Null; + p->aMem[n].db = db; } } -#endif + p->explain = pParse->explain; + sqlite3VdbeRewind(p); } /* @@ -58805,6 +59378,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( !zMaster ){ return SQLITE_NOMEM; } + sqlite3FileSuffix3(zMainFile, zMaster); rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); }while( rc==SQLITE_OK && res ); if( rc==SQLITE_OK ){ @@ -59019,6 +59593,15 @@ SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ db->nStatement--; p->iStatement = 0; + if( rc==SQLITE_OK ){ + if( eOp==SAVEPOINT_ROLLBACK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint); + } + if( rc==SQLITE_OK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint); + } + } + /* If the statement transaction is being rolled back, also restore the ** database handles deferred constraint counter to the value it had when ** the statement transaction was opened. */ @@ -59198,17 +59781,11 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ ** do so. If this operation returns an error, and the current statement ** error code is SQLITE_OK or SQLITE_CONSTRAINT, then promote the ** current statement error code. - ** - ** Note that sqlite3VdbeCloseStatement() can only fail if eStatementOp - ** is SAVEPOINT_ROLLBACK. But if p->rc==SQLITE_OK then eStatementOp - ** must be SAVEPOINT_RELEASE. Hence the NEVER(p->rc==SQLITE_OK) in - ** the following code. */ if( eStatementOp ){ rc = sqlite3VdbeCloseStatement(p, eStatementOp); if( rc ){ - assert( eStatementOp==SAVEPOINT_ROLLBACK ); - if( NEVER(p->rc==SQLITE_OK) || p->rc==SQLITE_CONSTRAINT ){ + if( p->rc==SQLITE_OK || p->rc==SQLITE_CONSTRAINT ){ p->rc = rc; sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; @@ -59401,6 +59978,7 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(VdbeFunc *pVdbeFunc, int mask){ */ SQLITE_PRIVATE void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ SubProgram *pSub, *pNext; + int i; assert( p->db==0 || p->db==db ); releaseMemArray(p->aVar, p->nVar); releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); @@ -59409,6 +59987,7 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); sqlite3DbFree(db, pSub); } + for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]); vdbeFreeOpArray(db, p->aOp, p->nOp); sqlite3DbFree(db, p->aLabel); sqlite3DbFree(db, p->aColName); @@ -59854,7 +60433,7 @@ SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeRecordUnpack( idx += getVarint32(&aKey[idx], serial_type); pMem->enc = pKeyInfo->enc; pMem->db = pKeyInfo->db; - pMem->flags = 0; + /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */ pMem->zMalloc = 0; d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); pMem++; @@ -59869,6 +60448,7 @@ SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeRecordUnpack( ** This routine destroys a UnpackedRecord object. */ SQLITE_PRIVATE void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord *p){ +#ifdef SQLITE_DEBUG int i; Mem *pMem; @@ -59882,6 +60462,7 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord *p){ */ if( NEVER(pMem->zMalloc) ) sqlite3VdbeMemRelease(pMem); } +#endif if( p->flags & UNPACKED_NEED_FREE ){ sqlite3DbFree(p->pKeyInfo->db, p); } @@ -59935,7 +60516,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompare( /* Compilers may complain that mem1.u.i is potentially uninitialized. ** We could initialize it, as shown here, to silence those complaints. - ** But in fact, mem1.u.i will never actually be used initialized, and doing + ** But in fact, mem1.u.i will never actually be used uninitialized, and doing ** the unnecessary initialization has a measurable negative performance ** impact, since this routine is a very high runner. And so, we choose ** to ignore the compiler warnings and leave this variable uninitialized. @@ -60317,7 +60898,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt){ Vdbe *v = (Vdbe*)pStmt; sqlite3_mutex_enter(v->db->mutex); rc = sqlite3VdbeReset(v); - sqlite3VdbeMakeReady(v, -1, 0, 0, 0, 0, 0); + sqlite3VdbeRewind(v); assert( (rc & (v->db->errMask))==rc ); rc = sqlite3ApiExit(v->db, rc); sqlite3_mutex_leave(v->db->mutex); @@ -60674,6 +61255,14 @@ end_of_step: return (rc&db->errMask); } +/* +** The maximum number of times that a statement will try to reparse +** itself before giving up and returning SQLITE_SCHEMA. +*/ +#ifndef SQLITE_MAX_SCHEMA_RETRY +# define SQLITE_MAX_SCHEMA_RETRY 5 +#endif + /* ** This is the top-level implementation of sqlite3_step(). Call ** sqlite3Step() to do most of the work. If a schema error occurs, @@ -60692,7 +61281,7 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ db = v->db; sqlite3_mutex_enter(db->mutex); while( (rc = sqlite3Step(v))==SQLITE_SCHEMA - && cnt++ < 5 + && cnt++ < SQLITE_MAX_SCHEMA_RETRY && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ sqlite3_reset(pStmt); v->expired = 0; @@ -61382,32 +61971,6 @@ SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ return p ? p->nVar : 0; } -/* -** Create a mapping from variable numbers to variable names -** in the Vdbe.azVar[] array, if such a mapping does not already -** exist. -*/ -static void createVarMap(Vdbe *p){ - if( !p->okVar ){ - int j; - Op *pOp; - sqlite3_mutex_enter(p->db->mutex); - /* The race condition here is harmless. If two threads call this - ** routine on the same Vdbe at the same time, they both might end - ** up initializing the Vdbe.azVar[] array. That is a little extra - ** work but it results in the same answer. - */ - for(j=0, pOp=p->aOp; jnOp; j++, pOp++){ - if( pOp->opcode==OP_Variable ){ - assert( pOp->p1>0 && pOp->p1<=p->nVar ); - p->azVar[pOp->p1-1] = pOp->p4.z; - } - } - p->okVar = 1; - sqlite3_mutex_leave(p->db->mutex); - } -} - /* ** Return the name of a wildcard parameter. Return NULL if the index ** is out of range or if the wildcard is unnamed. @@ -61416,10 +61979,9 @@ static void createVarMap(Vdbe *p){ */ SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ Vdbe *p = (Vdbe*)pStmt; - if( p==0 || i<1 || i>p->nVar ){ + if( p==0 || i<1 || i>p->nzVar ){ return 0; } - createVarMap(p); return p->azVar[i-1]; } @@ -61433,9 +61995,8 @@ SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nNa if( p==0 ){ return 0; } - createVarMap(p); if( zName ){ - for(i=0; inVar; i++){ + for(i=0; inzVar; i++){ const char *z = p->azVar[i]; if( z && memcmp(z,zName,nName)==0 && z[nName]==0 ){ return i+1; @@ -62350,6 +62911,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( Mem *pOut = 0; /* Output operand */ int iCompare = 0; /* Result of last OP_Compare operation */ int *aPermute = 0; /* Permutation of columns for OP_Compare */ + i64 lastRowid = db->lastRowid; /* Saved value of the last insert ROWID */ #ifdef VDBE_PROFILE u64 start; /* CPU clock count at start of opcode */ int origPc; /* Program counter at start of opcode */ @@ -62758,6 +63320,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( } cm; struct OP_Trace_stack_vars { char *zTrace; + char *z; } cn; } u; /* End automatically generated code @@ -62991,7 +63554,7 @@ case OP_Yield: { /* in1 */ /* Opcode: HaltIfNull P1 P2 P3 P4 * ** -** Check the value in register P3. If is is NULL then Halt using +** Check the value in register P3. If it is NULL then Halt using ** parameter P1, P2, and P4 as if this were a Halt instruction. If the ** value in register P3 is not NULL, then this routine is a no-op. */ @@ -63028,6 +63591,7 @@ case OP_Halt: { p->nFrame--; sqlite3VdbeSetChanges(db, p->nChange); pc = sqlite3VdbeFrameRestore(pFrame); + lastRowid = db->lastRowid; if( pOp->p2==OE_Ignore ){ /* Instruction pc is the OP_Program that invoked the sub-program ** currently being halted. If the p2 instruction of this OP_Halt @@ -63183,6 +63747,7 @@ case OP_Variable: { /* out2-prerelease */ #endif /* local variables moved into u.ab */ assert( pOp->p1>0 && pOp->p1<=p->nVar ); + assert( pOp->p4.z==0 || pOp->p4.z==p->azVar[pOp->p1-1] ); u.ab.pVar = &p->aVar[pOp->p1 - 1]; if( sqlite3VdbeMemTooBig(u.ab.pVar) ){ goto too_big; @@ -63600,16 +64165,9 @@ case OP_Function: { assert( pOp[-1].opcode==OP_CollSeq ); u.ag.ctx.pColl = pOp[-1].p4.pColl; } + db->lastRowid = lastRowid; (*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); /* IMP: R-24505-23230 */ - if( db->mallocFailed ){ - /* Even though a malloc() has failed, the implementation of the - ** user function may have called an sqlite3_result_XXX() function - ** to return a value. The following call releases any resources - ** associated with such a value. - */ - sqlite3VdbeMemRelease(&u.ag.ctx.s); - goto no_mem; - } + lastRowid = db->lastRowid; /* If any auxiliary data functions have been called by this user function, ** immediately call the destructor for any non-static values. @@ -63620,6 +64178,16 @@ case OP_Function: { pOp->p4type = P4_VDBEFUNC; } + if( db->mallocFailed ){ + /* Even though a malloc() has failed, the implementation of the + ** user function may have called an sqlite3_result_XXX() function + ** to return a value. The following call releases any resources + ** associated with such a value. + */ + sqlite3VdbeMemRelease(&u.ag.ctx.s); + goto no_mem; + } + /* If the function returned an error, throw an exception */ if( u.ag.ctx.isError ){ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.ag.ctx.s)); @@ -63923,7 +64491,7 @@ case OP_ToReal: { /* same as TK_TO_REAL, in1 */ ** If SQLITE_NULLEQ is set in P5 then the result of comparison is always either ** true or false and is never NULL. If both operands are NULL then the result ** of comparison is false. If either operand is NULL then the result is true. -** If neither operand is NULL the the result is the same as it would be if +** If neither operand is NULL the result is the same as it would be if ** the SQLITE_NULLEQ flag were omitted from P5. */ /* Opcode: Eq P1 P2 P3 P4 P5 @@ -63935,7 +64503,7 @@ case OP_ToReal: { /* same as TK_TO_REAL, in1 */ ** If SQLITE_NULLEQ is set in P5 then the result of comparison is always either ** true or false and is never NULL. If both operands are NULL then the result ** of comparison is true. If either operand is NULL then the result is false. -** If neither operand is NULL the the result is the same as it would be if +** If neither operand is NULL the result is the same as it would be if ** the SQLITE_NULLEQ flag were omitted from P5. */ /* Opcode: Le P1 P2 P3 P4 P5 @@ -63973,7 +64541,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ pIn3 = &aMem[pOp->p3]; u.ai.flags1 = pIn1->flags; u.ai.flags3 = pIn3->flags; - if( (pIn1->flags | pIn3->flags)&MEM_Null ){ + if( (u.ai.flags1 | u.ai.flags3)&MEM_Null ){ /* One or both operands are NULL */ if( pOp->p5 & SQLITE_NULLEQ ){ /* If SQLITE_NULLEQ is set (which will only happen if the operator is @@ -63981,7 +64549,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** or not both operands are null. */ assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne ); - u.ai.res = (pIn1->flags & pIn3->flags & MEM_Null)==0; + u.ai.res = (u.ai.flags1 & u.ai.flags3 & MEM_Null)==0; }else{ /* SQLITE_NULLEQ is clear and at least one operand is NULL, ** then the result is always NULL. @@ -64220,13 +64788,13 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ /* Opcode: If P1 P2 P3 * * ** -** Jump to P2 if the value in register P1 is true. The value is +** Jump to P2 if the value in register P1 is true. The value ** is considered true if it is numeric and non-zero. If the value ** in P1 is NULL then take the jump if P3 is true. */ /* Opcode: IfNot P1 P2 P3 * * ** -** Jump to P2 if the value in register P1 is False. The value is +** Jump to P2 if the value in register P1 is False. The value ** is considered true if it has a numeric value of zero. If the value ** in P1 is NULL then take the jump if P3 is true. */ @@ -64807,6 +65375,17 @@ case OP_Savepoint: { }else{ u.aq.nName = sqlite3Strlen30(u.aq.zName); +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* This call is Ok even if this savepoint is actually a transaction + ** savepoint (and therefore should not prompt xSavepoint()) callbacks. + ** If this is a transaction savepoint being opened, it is guaranteed + ** that the db->aVTrans[] array is empty. */ + assert( db->autoCommit==0 || db->nVTrans==0 ); + rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, + db->nStatement+db->nSavepoint); + if( rc!=SQLITE_OK ) goto abort_due_to_error; +#endif + /* Create a new savepoint structure. */ u.aq.pNew = sqlite3DbMallocRaw(db, sizeof(Savepoint)+u.aq.nName+1); if( u.aq.pNew ){ @@ -64913,6 +65492,11 @@ case OP_Savepoint: { }else{ db->nDeferredCons = u.aq.pSavepoint->nDeferredCons; } + + if( !isTransaction ){ + rc = sqlite3VtabSavepoint(db, u.aq.p1, u.aq.iSavepoint); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + } } } @@ -65052,7 +65636,11 @@ case OP_Transaction: { db->nStatement++; p->iStatement = db->nSavepoint + db->nStatement; } - rc = sqlite3BtreeBeginStmt(u.as.pBt, p->iStatement); + + rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, p->iStatement-1); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginStmt(u.as.pBt, p->iStatement); + } /* Store the current value of the database handles deferred constraint ** counter. If the statement transaction needs to be rolled back, @@ -65373,7 +65961,7 @@ case OP_OpenEphemeral: { u.ax.pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( u.ax.pCx==0 ) goto no_mem; u.ax.pCx->nullRow = 1; - rc = sqlite3BtreeOpen(0, db, &u.ax.pCx->pBt, + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &u.ax.pCx->pBt, BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); if( rc==SQLITE_OK ){ rc = sqlite3BtreeBeginTrans(u.ax.pCx->pBt, 1); @@ -65860,7 +66448,7 @@ case OP_IsUnique: { /* jump, in3 */ /* Opcode: NotExists P1 P2 P3 * * ** -** Use the content of register P3 as a integer key. If a record +** Use the content of register P3 as an integer key. If a record ** with that key does not exist in table of P1, then jump to P2. ** If the record does exist, then fall through. The cursor is left ** pointing to the record if it exists. @@ -65938,7 +66526,7 @@ case OP_Sequence: { /* out2-prerelease */ ** If P3>0 then P3 is a register in the root frame of this VDBE that holds ** the largest previously generated record number. No new record numbers are ** allowed to be less than this value. When this value reaches its maximum, -** a SQLITE_FULL error is generated. The P3 register is updated with the ' +** an SQLITE_FULL error is generated. The P3 register is updated with the ' ** generated record number. This P3 mechanism is used to help implement the ** AUTOINCREMENT feature. */ @@ -66047,7 +66635,7 @@ case OP_NewRowid: { /* out2-prerelease */ assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is ** an AUTOINCREMENT table. */ /* on the first attempt, simply do one more than previous */ - u.be.v = db->lastRowid; + u.be.v = lastRowid; u.be.v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ u.be.v++; /* ensure non-zero */ u.be.cnt = 0; @@ -66159,7 +66747,7 @@ case OP_InsertInt: { } if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = u.bf.iKey; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = u.bf.iKey; if( u.bf.pData->flags & MEM_Null ){ u.bf.pData->z = 0; u.bf.pData->n = 0; @@ -66580,7 +67168,7 @@ case OP_Next: { /* jump */ /* Opcode: IdxInsert P1 P2 P3 * P5 ** -** Register P2 holds a SQL index key made using the +** Register P2 holds an SQL index key made using the ** MakeRecord instructions. This opcode writes that key ** into the index P1. Data for the entry is nil. ** @@ -67285,7 +67873,7 @@ case OP_Program: { /* jump */ p->nFrame++; u.by.pFrame->pParent = p->pFrame; - u.by.pFrame->lastRowid = db->lastRowid; + u.by.pFrame->lastRowid = lastRowid; u.by.pFrame->nChange = p->nChange; p->nChange = 0; p->pFrame = u.by.pFrame; @@ -68096,11 +68684,15 @@ case OP_VUpdate: { Mem *pX; #endif /* local variables moved into u.cm */ + assert( pOp->p2==1 || pOp->p5==OE_Fail || pOp->p5==OE_Rollback + || pOp->p5==OE_Abort || pOp->p5==OE_Ignore || pOp->p5==OE_Replace + ); u.cm.pVtab = pOp->p4.pVtab->pVtab; u.cm.pModule = (sqlite3_module *)u.cm.pVtab->pModule; u.cm.nArg = pOp->p2; assert( pOp->p4type==P4_VTAB ); if( ALWAYS(u.cm.pModule->xUpdate) ){ + u8 vtabOnConflict = db->vtabOnConflict; u.cm.apArg = p->apArg; u.cm.pX = &aMem[pOp->p3]; for(u.cm.i=0; u.cm.ivtabOnConflict = pOp->p5; rc = u.cm.pModule->xUpdate(u.cm.pVtab, u.cm.nArg, u.cm.apArg, &u.cm.rowid); + db->vtabOnConflict = vtabOnConflict; importVtabErrMsg(p, u.cm.pVtab); if( rc==SQLITE_OK && pOp->p1 ){ assert( u.cm.nArg>1 && u.cm.apArg[0] && (u.cm.apArg[0]->flags&MEM_Null) ); - db->lastRowid = u.cm.rowid; + db->lastRowid = lastRowid = u.cm.rowid; + } + if( rc==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ + if( pOp->p5==OE_Ignore ){ + rc = SQLITE_OK; + }else{ + p->errorAction = ((pOp->p5==OE_Replace) ? OE_Abort : pOp->p5); + } + }else{ + p->nChange++; } - p->nChange++; } break; } @@ -68168,21 +68770,21 @@ case OP_MaxPgcnt: { /* out2-prerelease */ case OP_Trace: { #if 0 /* local variables moved into u.cn */ char *zTrace; + char *z; #endif /* local variables moved into u.cn */ - u.cn.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql); - if( u.cn.zTrace ){ - if( db->xTrace ){ - char *z = sqlite3VdbeExpandSql(p, u.cn.zTrace); - db->xTrace(db->pTraceArg, z); - sqlite3DbFree(db, z); - } -#ifdef SQLITE_DEBUG - if( (db->flags & SQLITE_SqlTrace)!=0 ){ - sqlite3DebugPrintf("SQL-trace: %s\n", u.cn.zTrace); - } -#endif /* SQLITE_DEBUG */ + if( db->xTrace && (u.cn.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 ){ + u.cn.z = sqlite3VdbeExpandSql(p, u.cn.zTrace); + db->xTrace(db->pTraceArg, u.cn.z); + sqlite3DbFree(db, u.cn.z); } +#ifdef SQLITE_DEBUG + if( (db->flags & SQLITE_SqlTrace)!=0 + && (u.cn.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 + ){ + sqlite3DebugPrintf("SQL-trace: %s\n", u.cn.zTrace); + } +#endif /* SQLITE_DEBUG */ break; } #endif @@ -68266,6 +68868,7 @@ vdbe_error_halt: ** release the mutexes on btrees that were acquired at the ** top. */ vdbe_return: + db->lastRowid = lastRowid; sqlite3VdbeLeave(p); return rc; @@ -68606,7 +69209,10 @@ SQLITE_API int sqlite3_blob_open( sqlite3VdbeChangeP4(v, 3+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); sqlite3VdbeChangeP2(v, 7, pTab->nCol); if( !db->mallocFailed ){ - sqlite3VdbeMakeReady(v, 1, 1, 1, 0, 0, 0); + pParse->nVar = 1; + pParse->nMem = 1; + pParse->nTab = 1; + sqlite3VdbeMakeReady(v, pParse); } } @@ -71173,53 +71779,53 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ /* Wildcard of the form "?". Assign the next variable number */ assert( z[0]=='?' ); pExpr->iColumn = (ynVar)(++pParse->nVar); - }else if( z[0]=='?' ){ - /* Wildcard of the form "?nnn". Convert "nnn" to an integer and - ** use it as the variable number */ - i64 i; - int bOk = 0==sqlite3Atoi64(&z[1], &i, sqlite3Strlen30(&z[1]), SQLITE_UTF8); - pExpr->iColumn = (ynVar)i; - testcase( i==0 ); - testcase( i==1 ); - testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); - testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); - if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ - sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", - db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); - } - if( i>pParse->nVar ){ - pParse->nVar = (int)i; - } }else{ - /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable - ** number as the prior appearance of the same name, or if the name - ** has never appeared before, reuse the same variable number - */ - int i; - u32 n; - n = sqlite3Strlen30(z); - for(i=0; inVarExpr; i++){ - Expr *pE = pParse->apVarExpr[i]; - assert( pE!=0 ); - if( memcmp(pE->u.zToken, z, n)==0 && pE->u.zToken[n]==0 ){ - pExpr->iColumn = pE->iColumn; - break; + ynVar x = 0; + u32 n = sqlite3Strlen30(z); + if( z[0]=='?' ){ + /* Wildcard of the form "?nnn". Convert "nnn" to an integer and + ** use it as the variable number */ + i64 i; + int bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); + pExpr->iColumn = x = (ynVar)i; + testcase( i==0 ); + testcase( i==1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); + if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", + db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); + x = 0; } + if( i>pParse->nVar ){ + pParse->nVar = (int)i; + } + }else{ + /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable + ** number as the prior appearance of the same name, or if the name + ** has never appeared before, reuse the same variable number + */ + ynVar i; + for(i=0; inzVar; i++){ + if( pParse->azVar[i] && memcmp(pParse->azVar[i],z,n+1)==0 ){ + pExpr->iColumn = x = (ynVar)i+1; + break; + } + } + if( x==0 ) x = pExpr->iColumn = (ynVar)(++pParse->nVar); } - if( i>=pParse->nVarExpr ){ - pExpr->iColumn = (ynVar)(++pParse->nVar); - if( pParse->nVarExpr>=pParse->nVarExprAlloc-1 ){ - pParse->nVarExprAlloc += pParse->nVarExprAlloc + 10; - pParse->apVarExpr = - sqlite3DbReallocOrFree( - db, - pParse->apVarExpr, - pParse->nVarExprAlloc*sizeof(pParse->apVarExpr[0]) - ); + if( x>0 ){ + if( x>pParse->nzVar ){ + char **a; + a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); + if( a==0 ) return; /* Error reported through db->mallocFailed */ + pParse->azVar = a; + memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); + pParse->nzVar = x; } - if( !db->mallocFailed ){ - assert( pParse->apVarExpr!=0 ); - pParse->apVarExpr[pParse->nVarExpr++] = pExpr; + if( z[0]!='?' || pParse->azVar[x-1]==0 ){ + sqlite3DbFree(db, pParse->azVar[x-1]); + pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); } } } @@ -72963,7 +73569,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( pExpr->u.zToken[0]!=0 ); sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); if( pExpr->u.zToken[1]!=0 ){ - sqlite3VdbeChangeP4(v, -1, pExpr->u.zToken, P4_TRANSIENT); + assert( pExpr->u.zToken[0]=='?' + || strcmp(pExpr->u.zToken, pParse->azVar[pExpr->iColumn-1])==0 ); + sqlite3VdbeChangeP4(v, -1, pParse->azVar[pExpr->iColumn-1], P4_STATIC); } break; } @@ -74733,14 +75341,14 @@ static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){ /* Reload the table, index and permanent trigger schemas. */ zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName); if( !zWhere ) return; - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); #ifndef SQLITE_OMIT_TRIGGER /* Now, if the table is not stored in the temp database, reload any temp ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. */ if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ - sqlite3VdbeAddOp4(v, OP_ParseSchema, 1, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, 1, zWhere); } #endif } @@ -75994,8 +76602,12 @@ static void attachFunc( sqlite3 *db = sqlite3_context_db_handle(context); const char *zName; const char *zFile; + char *zPath = 0; + char *zErr = 0; + unsigned int flags; Db *aNew; char *zErrDyn = 0; + sqlite3_vfs *pVfs; UNUSED_PARAMETER(NotUsed); @@ -76048,8 +76660,18 @@ static void attachFunc( ** it to obtain the database schema. At this point the schema may ** or may not be initialised. */ - rc = sqlite3BtreeOpen(zFile, db, &aNew->pBt, 0, - db->openFlags | SQLITE_OPEN_MAIN_DB); + flags = db->openFlags; + rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + assert( pVfs ); + flags |= SQLITE_OPEN_MAIN_DB; + rc = sqlite3BtreeOpen(pVfs, zPath, db, &aNew->pBt, 0, flags); + sqlite3_free( zPath ); db->nDb++; if( rc==SQLITE_CONSTRAINT ){ rc = SQLITE_ERROR; @@ -76920,9 +77542,7 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; - sqlite3VdbeMakeReady(v, pParse->nVar, pParse->nMem, - pParse->nTab, pParse->nMaxArg, pParse->explain, - pParse->isMultiWrite && pParse->mayAbort); + sqlite3VdbeMakeReady(v, pParse); pParse->rc = SQLITE_DONE; pParse->colNamesSet = 0; }else{ @@ -78341,8 +78961,8 @@ SQLITE_PRIVATE void sqlite3EndTable( #endif /* Reparse everything to update our internal data structures */ - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, - sqlite3MPrintf(db, "tbl_name='%q'",p->zName), P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "tbl_name='%q'", p->zName)); } @@ -79539,9 +80159,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( if( pTblName ){ sqlite3RefillIndex(pParse, pIndex, iMem); sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, - sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), - P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); sqlite3VdbeAddOp1(v, OP_Expire, 0); } } @@ -80163,7 +80782,7 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ SQLITE_OPEN_DELETEONCLOSE | SQLITE_OPEN_TEMP_DB; - rc = sqlite3BtreeOpen(0, db, &pBt, 0, flags); + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pBt, 0, flags); if( rc!=SQLITE_OK ){ sqlite3ErrorMsg(pParse, "unable to open a temporary database " "file for storing temporary tables"); @@ -81339,6 +81958,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iRowid, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, OE_Abort); sqlite3MayAbort(pParse); }else #endif @@ -81573,8 +82193,14 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( } } if( doMakeRec ){ + const char *zAff; + if( pTab->pSelect || (pParse->db->flags & SQLITE_IdxRealAsInt)!=0 ){ + zAff = 0; + }else{ + zAff = sqlite3IndexAffinityStr(v, pIdx); + } sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol+1, regOut); - sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), P4_TRANSIENT); + sqlite3VdbeChangeP4(v, -1, zAff, P4_TRANSIENT); } sqlite3ReleaseTempRange(pParse, regBase, nCol+1); return regBase; @@ -82086,10 +82712,10 @@ struct compareInfo { ** whereas only characters less than 0x80 do in ASCII. */ #if defined(SQLITE_EBCDIC) -# define sqlite3Utf8Read(A,C) (*(A++)) -# define GlogUpperToLower(A) A = sqlite3UpperToLower[A] +# define sqlite3Utf8Read(A,C) (*(A++)) +# define GlogUpperToLower(A) A = sqlite3UpperToLower[A] #else -# define GlogUpperToLower(A) if( A<0x80 ){ A = sqlite3UpperToLower[A]; } +# define GlogUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; } #endif static const struct compareInfo globInfo = { '*', '?', '[', 0 }; @@ -82132,9 +82758,9 @@ static int patternCompare( const u8 *zPattern, /* The glob pattern */ const u8 *zString, /* The string to compare against the glob */ const struct compareInfo *pInfo, /* Information about how to do the compare */ - const int esc /* The escape character */ + u32 esc /* The escape character */ ){ - int c, c2; + u32 c, c2; int invert; int seen; u8 matchOne = pInfo->matchOne; @@ -82188,7 +82814,7 @@ static int patternCompare( return 0; } }else if( c==matchSet ){ - int prior_c = 0; + u32 prior_c = 0; assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ seen = 0; invert = 0; @@ -82264,7 +82890,7 @@ static void likeFunc( sqlite3_value **argv ){ const unsigned char *zA, *zB; - int escape = 0; + u32 escape = 0; int nPat; sqlite3 *db = sqlite3_context_db_handle(context); @@ -82354,6 +82980,21 @@ static void sourceidFunc( sqlite3_result_text(context, sqlite3_sourceid(), -1, SQLITE_STATIC); } +/* +** Implementation of the sqlite_log() function. This is a wrapper around +** sqlite3_log(). The return value is NULL. The function exists purely for +** its side-effects. +*/ +static void errlogFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(context); + sqlite3_log(sqlite3_value_int(argv[0]), "%s", sqlite3_value_text(argv[1])); +} + /* ** Implementation of the sqlite_compileoption_used() function. ** The result is an integer that identifies if the compiler option @@ -83121,6 +83762,7 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ FUNCTION(nullif, 2, 0, 1, nullifFunc ), FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), + FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), @@ -83559,13 +84201,25 @@ static void fkLookupParent( /* If the parent table is the same as the child table, and we are about ** to increment the constraint-counter (i.e. this is an INSERT operation), ** then check if the row being inserted matches itself. If so, do not - ** increment the constraint-counter. */ + ** increment the constraint-counter. + ** + ** If any of the parent-key values are NULL, then the row cannot match + ** itself. So set JUMPIFNULL to make sure we do the OP_Found if any + ** of the parent-key values are NULL (at this point it is known that + ** none of the child key values are). + */ if( pTab==pFKey->pFrom && nIncr==1 ){ int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; for(i=0; iaiColumn[i]+1+regData; + assert( aiCol[i]!=pTab->iPKey ); + if( pIdx->aiColumn[i]==pTab->iPKey ){ + /* The parent key is a composite key that includes the IPK column */ + iParent = regData; + } sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); + sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); } sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); } @@ -85333,6 +85987,7 @@ SQLITE_PRIVATE void sqlite3Insert( const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 1, pTab->nCol+2, regIns, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); }else #endif @@ -86098,6 +86753,18 @@ static int xferOptimization( return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif +#ifndef SQLITE_OMIT_FOREIGN_KEY + /* Disallow the transfer optimization if the destination table constains + ** any foreign key constraints. This is more restrictive than necessary. + ** But the main beneficiary of the transfer optimization is the VACUUM + ** command, and the VACUUM command disables foreign key constraints. So + ** the extra complication to make this rule less restrictive is probably + ** not worth the effort. Ticket [6284df89debdfa61db8073e062908af0c9b6118e] + */ + if( (pParse->db->flags & SQLITE_ForeignKeys)!=0 && pDest->pFKey!=0 ){ + return 0; + } +#endif /* If we get this far, it means either: ** @@ -87430,10 +88097,6 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ ** This file contains code used to implement the PRAGMA command. */ -/* Ignore this whole file if pragmas are disabled -*/ -#if !defined(SQLITE_OMIT_PRAGMA) - /* ** Interpret the given string as a safety level. Return 0 for OFF, ** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or @@ -87466,10 +88129,16 @@ static u8 getSafetyLevel(const char *z){ /* ** Interpret the given string as a boolean value. */ -static u8 getBoolean(const char *z){ +SQLITE_PRIVATE u8 sqlite3GetBoolean(const char *z){ return getSafetyLevel(z)&1; } +/* The sqlite3GetBoolean() function is used by other modules but the +** remainder of this file is specific to PRAGMA processing. So omit +** the rest of the file if PRAGMAs are omitted from the build. +*/ +#if !defined(SQLITE_OMIT_PRAGMA) + /* ** Interpret the given string as a locking mode value. */ @@ -87636,7 +88305,7 @@ static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ mask &= ~(SQLITE_ForeignKeys); } - if( getBoolean(zRight) ){ + if( sqlite3GetBoolean(zRight) ){ db->flags |= mask; }else{ db->flags &= ~mask; @@ -87850,7 +88519,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int b = -1; assert( pBt!=0 ); if( zRight ){ - b = getBoolean(zRight); + b = sqlite3GetBoolean(zRight); } if( pId2->n==0 && b>=0 ){ int ii; @@ -88450,7 +89119,7 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef NDEBUG if( sqlite3StrICmp(zLeft, "parser_trace")==0 ){ if( zRight ){ - if( getBoolean(zRight) ){ + if( sqlite3GetBoolean(zRight) ){ sqlite3ParserTrace(stderr, "parser: "); }else{ sqlite3ParserTrace(0, 0); @@ -88464,7 +89133,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ if( sqlite3StrICmp(zLeft, "case_sensitive_like")==0 ){ if( zRight ){ - sqlite3RegisterLikeFunctions(db, getBoolean(zRight)); + sqlite3RegisterLikeFunctions(db, sqlite3GetBoolean(zRight)); } }else @@ -94042,11 +94711,13 @@ SQLITE_PRIVATE int sqlite3Select( ** and pKeyInfo to the KeyInfo structure required to navigate the ** index. ** + ** (2011-04-15) Do not do a full scan of an unordered index. + ** ** In practice the KeyInfo structure will not be used. It is only ** passed to keep OP_OpenRead happy. */ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( !pBest || pIdx->nColumnnColumn ){ + if( pIdx->bUnordered==0 && (!pBest || pIdx->nColumnnColumn) ){ pBest = pIdx; } } @@ -94768,9 +95439,8 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( pTrig->table, z); sqlite3DbFree(db, z); sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, sqlite3MPrintf( - db, "type='trigger' AND name='%q'", zName), P4_DYNAMIC - ); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName)); } if( db->init.busy ){ @@ -95603,7 +96273,8 @@ static void updateVirtualTable( ExprList *pChanges, /* The columns to change in the UPDATE statement */ Expr *pRowidExpr, /* Expression used to recompute the rowid */ int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ - Expr *pWhere /* WHERE clause of the UPDATE statement */ + Expr *pWhere, /* WHERE clause of the UPDATE statement */ + int onError /* ON CONFLICT strategy */ ); #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -95823,7 +96494,7 @@ SQLITE_PRIVATE void sqlite3Update( } for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int reg; - if( chngRowid ){ + if( hasFK || chngRowid ){ reg = ++pParse->nMem; }else{ reg = 0; @@ -95847,7 +96518,7 @@ SQLITE_PRIVATE void sqlite3Update( /* Virtual tables must be handled separately */ if( IsVirtual(pTab) ){ updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, - pWhere); + pWhere, onError); pWhere = 0; pTabList = 0; goto update_cleanup; @@ -96177,7 +96848,8 @@ static void updateVirtualTable( ExprList *pChanges, /* The columns to change in the UPDATE statement */ Expr *pRowid, /* Expression used to recompute the rowid */ int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ - Expr *pWhere /* WHERE clause of the UPDATE statement */ + Expr *pWhere, /* WHERE clause of the UPDATE statement */ + int onError /* ON CONFLICT strategy */ ){ Vdbe *v = pParse->pVdbe; /* Virtual machine under construction */ ExprList *pEList = 0; /* The result set of the SELECT statement */ @@ -96234,6 +96906,7 @@ static void updateVirtualTable( } sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 0, pTab->nCol+2, iReg, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); sqlite3VdbeJumpHere(v, addr); @@ -96607,6 +97280,18 @@ end_of_vacuum: */ #ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Before a virtual table xCreate() or xConnect() method is invoked, the +** sqlite3.pVtabCtx member variable is set to point to an instance of +** this struct allocated on the stack. It is used by the implementation of +** the sqlite3_declare_vtab() and sqlite3_vtab_config() APIs, both of which +** are invoked only from within xCreate and xConnect methods. +*/ +struct VtabCtx { + Table *pTab; + VTable *pVTable; +}; + /* ** The actual function that does the work of creating a new module. ** This function implements the sqlite3_create_module() and @@ -96635,13 +97320,13 @@ static int createModule( pMod->xDestroy = xDestroy; pDel = (Module *)sqlite3HashInsert(&db->aModule, zCopy, nName, (void*)pMod); if( pDel && pDel->xDestroy ){ + sqlite3ResetInternalSchema(db, -1); pDel->xDestroy(pDel->pAux); } sqlite3DbFree(db, pDel); if( pDel==pMod ){ db->mallocFailed = 1; } - sqlite3ResetInternalSchema(db, -1); }else if( xDestroy ){ xDestroy(pAux); } @@ -96964,7 +97649,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, pTab->zName, sqlite3Strlen30(pTab->zName) + 1); } @@ -97027,6 +97712,7 @@ static int vtabCallConstructor( int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), char **pzErr ){ + VtabCtx sCtx; VTable *pVTable; int rc; const char *const*azArg = (const char *const*)pTab->azModuleArg; @@ -97046,12 +97732,14 @@ static int vtabCallConstructor( pVTable->db = db; pVTable->pMod = pMod; - assert( !db->pVTab ); - assert( xConstruct ); - db->pVTab = pTab; - /* Invoke the virtual table constructor */ + assert( &db->pVtabCtx ); + assert( xConstruct ); + sCtx.pTab = pTab; + sCtx.pVTable = pVTable; + db->pVtabCtx = &sCtx; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + db->pVtabCtx = 0; if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; if( SQLITE_OK!=rc ){ @@ -97067,7 +97755,7 @@ static int vtabCallConstructor( ** the sqlite3_vtab object if successful. */ pVTable->pVtab->pModule = pMod->pModule; pVTable->nRef = 1; - if( db->pVTab ){ + if( sCtx.pTab ){ const char *zFormat = "vtable constructor did not declare schema: %s"; *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); sqlite3VtabUnlock(pVTable); @@ -97115,7 +97803,6 @@ static int vtabCallConstructor( } sqlite3DbFree(db, zModuleName); - db->pVTab = 0; return rc; } @@ -97156,11 +97843,11 @@ SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ return rc; } - /* -** Add the virtual table pVTab to the array sqlite3.aVTrans[]. +** Grow the db->aVTrans[] array so that there is room for at least one +** more v-table. Return SQLITE_NOMEM if a malloc fails, or SQLITE_OK otherwise. */ -static int addToVTrans(sqlite3 *db, VTable *pVTab){ +static int growVTrans(sqlite3 *db){ const int ARRAY_INCR = 5; /* Grow the sqlite3.aVTrans array if required */ @@ -97175,10 +97862,17 @@ static int addToVTrans(sqlite3 *db, VTable *pVTab){ db->aVTrans = aVTrans; } + return SQLITE_OK; +} + +/* +** Add the virtual table pVTab to the array sqlite3.aVTrans[]. Space should +** have already been reserved using growVTrans(). +*/ +static void addToVTrans(sqlite3 *db, VTable *pVTab){ /* Add pVtab to the end of sqlite3.aVTrans */ db->aVTrans[db->nVTrans++] = pVTab; sqlite3VtabLock(pVTab); - return SQLITE_OK; } /* @@ -97216,7 +97910,10 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, /* Justification of ALWAYS(): The xConstructor method is required to ** create a valid sqlite3_vtab if it returns SQLITE_OK. */ if( rc==SQLITE_OK && ALWAYS(sqlite3GetVTable(db, pTab)) ){ - rc = addToVTrans(db, sqlite3GetVTable(db, pTab)); + rc = growVTrans(db); + if( rc==SQLITE_OK ){ + addToVTrans(db, sqlite3GetVTable(db, pTab)); + } } return rc; @@ -97235,8 +97932,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ char *zErr = 0; sqlite3_mutex_enter(db->mutex); - pTab = db->pVTab; - if( !pTab ){ + if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ sqlite3Error(db, SQLITE_MISUSE, 0); sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; @@ -97263,7 +97959,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ pParse->pNewTable->nCol = 0; pParse->pNewTable->aCol = 0; } - db->pVTab = 0; + db->pVtabCtx->pTab = 0; }else{ sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); @@ -97333,6 +98029,7 @@ static void callFinaliser(sqlite3 *db, int offset){ x = *(int (**)(sqlite3_vtab *))((char *)p->pModule + offset); if( x ) x(p); } + pVTab->iSavepoint = 0; sqlite3VtabUnlock(pVTab); } sqlite3DbFree(db, db->aVTrans); @@ -97415,7 +98112,6 @@ SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ if( pModule->xBegin ){ int i; - /* If pVtab is already in the aVTrans array, return early */ for(i=0; inVTrans; i++){ if( db->aVTrans[i]==pVTab ){ @@ -97423,10 +98119,62 @@ SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ } } - /* Invoke the xBegin method */ - rc = pModule->xBegin(pVTab->pVtab); + /* Invoke the xBegin method. If successful, add the vtab to the + ** sqlite3.aVTrans[] array. */ + rc = growVTrans(db); if( rc==SQLITE_OK ){ - rc = addToVTrans(db, pVTab); + rc = pModule->xBegin(pVTab->pVtab); + if( rc==SQLITE_OK ){ + addToVTrans(db, pVTab); + } + } + } + return rc; +} + +/* +** Invoke either the xSavepoint, xRollbackTo or xRelease method of all +** virtual tables that currently have an open transaction. Pass iSavepoint +** as the second argument to the virtual table method invoked. +** +** If op is SAVEPOINT_BEGIN, the xSavepoint method is invoked. If it is +** SAVEPOINT_ROLLBACK, the xRollbackTo method. Otherwise, if op is +** SAVEPOINT_RELEASE, then the xRelease method of each virtual table with +** an open transaction is invoked. +** +** If any virtual table method returns an error code other than SQLITE_OK, +** processing is abandoned and the error returned to the caller of this +** function immediately. If all calls to virtual table methods are successful, +** SQLITE_OK is returned. +*/ +SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ + int rc = SQLITE_OK; + + assert( op==SAVEPOINT_RELEASE||op==SAVEPOINT_ROLLBACK||op==SAVEPOINT_BEGIN ); + assert( iSavepoint>=0 ); + if( db->aVTrans ){ + int i; + for(i=0; rc==SQLITE_OK && inVTrans; i++){ + VTable *pVTab = db->aVTrans[i]; + const sqlite3_module *pMod = pVTab->pMod->pModule; + if( pMod->iVersion>=2 ){ + int (*xMethod)(sqlite3_vtab *, int); + switch( op ){ + case SAVEPOINT_BEGIN: + xMethod = pMod->xSavepoint; + pVTab->iSavepoint = iSavepoint+1; + break; + case SAVEPOINT_ROLLBACK: + xMethod = pMod->xRollbackTo; + break; + default: + xMethod = pMod->xRelease; + break; + } + if( xMethod && pVTab->iSavepoint>iSavepoint ){ + rc = xMethod(db->aVTrans[i]->pVtab, iSavepoint); + } + } } } return rc; @@ -97530,6 +98278,57 @@ SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ } } +/* +** Return the ON CONFLICT resolution mode in effect for the virtual +** table update operation currently in progress. +** +** The results of this routine are undefined unless it is called from +** within an xUpdate method. +*/ +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *db){ + static const unsigned char aMap[] = { + SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE + }; + assert( OE_Rollback==1 && OE_Abort==2 && OE_Fail==3 ); + assert( OE_Ignore==4 && OE_Replace==5 ); + assert( db->vtabOnConflict>=1 && db->vtabOnConflict<=5 ); + return (int)aMap[db->vtabOnConflict-1]; +} + +/* +** Call from within the xCreate() or xConnect() methods to provide +** the SQLite core with additional information about the behavior +** of the virtual table being implemented. +*/ +SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc = SQLITE_OK; + + sqlite3_mutex_enter(db->mutex); + + va_start(ap, op); + switch( op ){ + case SQLITE_VTAB_CONSTRAINT_SUPPORT: { + VtabCtx *p = db->pVtabCtx; + if( !p ){ + rc = SQLITE_MISUSE_BKPT; + }else{ + assert( p->pTab==0 || (p->pTab->tabFlags & TF_Virtual)!=0 ); + p->pVTable->bConstraint = (u8)va_arg(ap, int); + } + break; + } + default: + rc = SQLITE_MISUSE_BKPT; + break; + } + va_end(ap); + + if( rc!=SQLITE_OK ) sqlite3Error(db, rc, 0); + sqlite3_mutex_leave(db->mutex); + return rc; +} + #endif /* SQLITE_OMIT_VIRTUALTABLE */ /************** End of vtab.c ************************************************/ @@ -106577,13 +107376,12 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ testcase( z[0]=='x' ); testcase( z[0]=='X' ); if( z[1]=='\'' ){ *tokenType = TK_BLOB; - for(i=2; (c=z[i])!=0 && c!='\''; i++){ - if( !sqlite3Isxdigit(c) ){ - *tokenType = TK_ILLEGAL; - } + for(i=2; sqlite3Isxdigit(z[i]); i++){} + if( z[i]!='\'' || i%2 ){ + *tokenType = TK_ILLEGAL; + while( z[i] && z[i]!='\'' ){ i++; } } - if( i%2 || !c ) *tokenType = TK_ILLEGAL; - if( c ) i++; + if( z[i] ) i++; return i; } /* Otherwise fall through to the next case */ @@ -106636,9 +107434,8 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr assert( pParse->pNewTable==0 ); assert( pParse->pNewTrigger==0 ); assert( pParse->nVar==0 ); - assert( pParse->nVarExpr==0 ); - assert( pParse->nVarExprAlloc==0 ); - assert( pParse->apVarExpr==0 ); + assert( pParse->nzVar==0 ); + assert( pParse->azVar==0 ); enableLookaside = db->lookaside.bEnabled; if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; while( !db->mallocFailed && zSql[i]!=0 ){ @@ -106732,7 +107529,8 @@ abort_parse: } sqlite3DeleteTrigger(db, pParse->pNewTrigger); - sqlite3DbFree(db, pParse->apVarExpr); + for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); + sqlite3DbFree(db, pParse->azVar); sqlite3DbFree(db, pParse->aAlias); while( pParse->pAinc ){ AutoincInfo *p = pParse->pAinc; @@ -107552,6 +108350,11 @@ SQLITE_API int sqlite3_config(int op, ...){ break; } + case SQLITE_CONFIG_URI: { + sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + break; + } + default: { rc = SQLITE_ERROR; break; @@ -108911,6 +109714,236 @@ SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ return oldLimit; /* IMP: R-53341-35419 */ } +/* +** This function is used to parse both URIs and non-URI filenames passed by the +** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database +** URIs specified as part of ATTACH statements. +** +** The first argument to this function is the name of the VFS to use (or +** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" +** query parameter. The second argument contains the URI (or non-URI filename) +** itself. When this function is called the *pFlags variable should contain +** the default flags to open the database handle with. The value stored in +** *pFlags may be updated before returning if the URI filename contains +** "cache=xxx" or "mode=xxx" query parameters. +** +** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to +** the VFS that should be used to open the database file. *pzFile is set to +** point to a buffer containing the name of the file to open. It is the +** responsibility of the caller to eventually call sqlite3_free() to release +** this buffer. +** +** If an error occurs, then an SQLite error code is returned and *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to eventually release +** this buffer by calling sqlite3_free(). +*/ +SQLITE_PRIVATE int sqlite3ParseUri( + const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ + const char *zUri, /* Nul-terminated URI to parse */ + unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ + sqlite3_vfs **ppVfs, /* OUT: VFS to use */ + char **pzFile, /* OUT: Filename component of URI */ + char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ +){ + int rc = SQLITE_OK; + unsigned int flags = *pFlags; + const char *zVfs = zDefaultVfs; + char *zFile; + char c; + int nUri = sqlite3Strlen30(zUri); + + assert( *pzErrMsg==0 ); + + if( ((flags & SQLITE_OPEN_URI) || sqlite3GlobalConfig.bOpenUri) + && nUri>=5 && memcmp(zUri, "file:", 5)==0 + ){ + char *zOpt; + int eState; /* Parser state when parsing URI */ + int iIn; /* Input character index */ + int iOut = 0; /* Output character index */ + int nByte = nUri+2; /* Bytes of space to allocate */ + + /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen + ** method that there may be extra parameters following the file-name. */ + flags |= SQLITE_OPEN_URI; + + for(iIn=0; iIn=0 && octet<256 ); + if( octet==0 ){ + /* This branch is taken when "%00" appears within the URI. In this + ** case we ignore all text in the remainder of the path, name or + ** value currently being parsed. So ignore the current character + ** and skip to the next "?", "=" or "&", as appropriate. */ + while( (c = zUri[iIn])!=0 && c!='#' + && (eState!=0 || c!='?') + && (eState!=1 || (c!='=' && c!='&')) + && (eState!=2 || c!='&') + ){ + iIn++; + } + continue; + } + c = octet; + }else if( eState==1 && (c=='&' || c=='=') ){ + if( zFile[iOut-1]==0 ){ + /* An empty option name. Ignore this option altogether. */ + while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; + continue; + } + if( c=='&' ){ + zFile[iOut++] = '\0'; + }else{ + eState = 2; + } + c = 0; + }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ + c = 0; + eState = 1; + } + zFile[iOut++] = c; + } + if( eState==1 ) zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + + /* Check if there were any options specified that should be interpreted + ** here. Options that are interpreted here include "vfs" and those that + ** correspond to flags that may be passed to the sqlite3_open_v2() + ** method. */ + zOpt = &zFile[sqlite3Strlen30(zFile)+1]; + while( zOpt[0] ){ + int nOpt = sqlite3Strlen30(zOpt); + char *zVal = &zOpt[nOpt+1]; + int nVal = sqlite3Strlen30(zVal); + + if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ + zVfs = zVal; + }else{ + struct OpenMode { + const char *z; + int mode; + } *aMode = 0; + char *zModeType = 0; + int mask = 0; + int limit = 0; + + if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ + static struct OpenMode aCacheMode[] = { + { "shared", SQLITE_OPEN_SHAREDCACHE }, + { "private", SQLITE_OPEN_PRIVATECACHE }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; + aMode = aCacheMode; + limit = mask; + zModeType = "cache"; + } + if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ + static struct OpenMode aOpenMode[] = { + { "ro", SQLITE_OPEN_READONLY }, + { "rw", SQLITE_OPEN_READWRITE }, + { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + aMode = aOpenMode; + limit = mask & flags; + zModeType = "access"; + } + + if( aMode ){ + int i; + int mode = 0; + for(i=0; aMode[i].z; i++){ + const char *z = aMode[i].z; + if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ + mode = aMode[i].mode; + break; + } + } + if( mode==0 ){ + *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); + rc = SQLITE_ERROR; + goto parse_uri_out; + } + if( mode>limit ){ + *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", + zModeType, zVal); + rc = SQLITE_PERM; + goto parse_uri_out; + } + flags = (flags & ~mask) | mode; + } + } + + zOpt = &zVal[nVal+1]; + } + + }else{ + zFile = sqlite3_malloc(nUri+2); + if( !zFile ) return SQLITE_NOMEM; + memcpy(zFile, zUri, nUri); + zFile[nUri] = '\0'; + zFile[nUri+1] = '\0'; + } + + *ppVfs = sqlite3_vfs_find(zVfs); + if( *ppVfs==0 ){ + *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); + rc = SQLITE_ERROR; + } + parse_uri_out: + if( rc!=SQLITE_OK ){ + sqlite3_free(zFile); + zFile = 0; + } + *pFlags = flags; + *pzFile = zFile; + return rc; +} + + /* ** This routine does the work of opening a database on behalf of ** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" @@ -108919,12 +109952,14 @@ SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ static int openDatabase( const char *zFilename, /* Database filename UTF-8 encoded */ sqlite3 **ppDb, /* OUT: Returned database handle */ - unsigned flags, /* Operational flags */ + unsigned int flags, /* Operational flags */ const char *zVfs /* Name of the VFS to use */ ){ - sqlite3 *db; - int rc; - int isThreadsafe; + sqlite3 *db; /* Store allocated handle here */ + int rc; /* Return code */ + int isThreadsafe; /* True for threadsafe connections */ + char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ + char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT @@ -108948,7 +109983,7 @@ static int openDatabase( testcase( (1<<(flags&7))==0x02 ); /* READONLY */ testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ - if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE; + if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE_BKPT; if( sqlite3GlobalConfig.bCoreMutex==0 ){ isThreadsafe = 0; @@ -109029,13 +110064,6 @@ static int openDatabase( sqlite3HashInit(&db->aModule); #endif - db->pVfs = sqlite3_vfs_find(zVfs); - if( !db->pVfs ){ - rc = SQLITE_ERROR; - sqlite3Error(db, rc, "no such vfs: %s", zVfs); - goto opendb_out; - } - /* Add the default collation sequence BINARY. BINARY works for both UTF-8 ** and UTF-16, so add a version for each to avoid any unnecessary ** conversions. The only error that can occur here is a malloc() failure. @@ -109058,9 +110086,18 @@ static int openDatabase( createCollation(db, "NOCASE", SQLITE_UTF8, SQLITE_COLL_NOCASE, 0, nocaseCollatingFunc, 0); - /* Open the backend database driver */ + /* Parse the filename/URI argument. */ db->openFlags = flags; - rc = sqlite3BtreeOpen(zFilename, db, &db->aDb[0].pBt, 0, + rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + sqlite3Error(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + sqlite3_free(zErrMsg); + goto opendb_out; + } + + /* Open the backend database driver */ + rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, flags | SQLITE_OPEN_MAIN_DB); if( rc!=SQLITE_OK ){ if( rc==SQLITE_IOERR_NOMEM ){ @@ -109153,6 +110190,7 @@ static int openDatabase( sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); opendb_out: + sqlite3_free(zOpen); if( db ){ assert( db->mutex!=0 || isThreadsafe==0 || sqlite3GlobalConfig.bFullMutex==0 ); sqlite3_mutex_leave(db->mutex); @@ -109184,7 +110222,7 @@ SQLITE_API int sqlite3_open_v2( int flags, /* Flags */ const char *zVfs /* Name of VFS module to use */ ){ - return openDatabase(filename, ppDb, flags, zVfs); + return openDatabase(filename, ppDb, (unsigned int)flags, zVfs); } #ifndef SQLITE_OMIT_UTF16 @@ -109789,12 +110827,45 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); + ** + ** If parameter onoff is non-zero, configure the wrappers so that all + ** subsequent calls to localtime() and variants fail. If onoff is zero, + ** undo this setting. + */ + case SQLITE_TESTCTRL_LOCALTIME_FAULT: { + sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); + break; + } + } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ return rc; } +/* +** This is a utility routine, useful to VFS implementations, that checks +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of the query parameter. +** +** The zFilename argument is the filename pointer passed into the xOpen() +** method of a VFS implementation. The zParam argument is the name of the +** query parameter we seek. This routine returns the value of the zParam +** parameter if it exists. If the parameter does not exist, this routine +** returns a NULL pointer. +*/ +SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] ){ + int x = strcmp(zFilename, zParam); + zFilename += sqlite3Strlen30(zFilename) + 1; + if( x==0 ) return zFilename; + zFilename += sqlite3Strlen30(zFilename) + 1; + } + return 0; +} + /************** End of main.c ************************************************/ /************** Begin file notify.c ******************************************/ /* @@ -110424,12 +111495,6 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ ** into a single segment. */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) -# define SQLITE_CORE 1 -#endif - /************** Include fts3Int.h in the middle of fts3.c ********************/ /************** Begin file fts3Int.h *****************************************/ /* @@ -110445,7 +111510,6 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ ****************************************************************************** ** */ - #ifndef _FTSINT_H #define _FTSINT_H @@ -110453,6 +111517,16 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ # define NDEBUG 1 #endif +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 +#endif + +#ifdef SQLITE_ENABLE_FTS3 /************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ /************** Begin file fts3_tokenizer.h **********************************/ /* @@ -110751,12 +111825,35 @@ SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const voi */ #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif + /* ** Maximum length of a varint encoded integer. The varint format is different ** from that used by SQLite, so the maximum length is 10, not 9. */ #define FTS3_VARINT_MAX 10 +/* +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. +** +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. +** +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. +*/ +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" + /* ** The testcase() macro is only used by the amalgamation. If undefined, ** make it a no-op. @@ -110796,22 +111893,43 @@ typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ typedef short int i16; /* 2-byte (or larger) signed integer */ typedef unsigned int u32; /* 4-byte unsigned integer */ typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ + /* ** Macro used to suppress compiler warnings for unused parameters. */ #define UNUSED_PARAMETER(x) (void)(x) + +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 #endif +/* +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) +#endif + +#endif /* SQLITE_AMALGAMATION */ + typedef struct Fts3Table Fts3Table; typedef struct Fts3Cursor Fts3Cursor; typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3PhraseToken Fts3PhraseToken; +typedef struct Fts3Doclist Fts3Doclist; typedef struct Fts3SegFilter Fts3SegFilter; typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; -typedef struct Fts3SegReaderCursor Fts3SegReaderCursor; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; /* ** A connection to a fulltext index is an instance of the following @@ -110832,7 +111950,7 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[24]; + sqlite3_stmt *aStmt[27]; char *zReadExprlist; char *zWriteExprlist; @@ -110840,21 +111958,43 @@ struct Fts3Table { int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ - /* The following hash table is used to buffer pending index updates during + /* TODO: Fix the first paragraph of this comment. + ** + ** The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the ** pending data, including hash table overhead, but not malloc overhead. ** When nPendingData exceeds nMaxPendingData, the buffer is flushed ** automatically. Variable iPrevDocid is the docid of the most recently ** inserted record. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. */ - int nMaxPendingData; - int nPendingData; - sqlite_int64 iPrevDocid; - Fts3Hash pendingTerms; + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ + +#if defined(SQLITE_DEBUG) + /* State variables used for validating that the transaction control + ** methods of the virtual table are called at appropriate times. These + ** values do not contribution to the FTS computation; they are used for + ** verifying the SQLite core. + */ + int inTransaction; /* True after xBegin but before xCommit/xRollback */ + int mxSavepoint; /* Largest valid xSavepoint integer */ +#endif }; /* @@ -110875,8 +112015,10 @@ struct Fts3Cursor { char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ + u8 bDesc; /* True to sort in descending order */ int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ + sqlite3_int64 nDoc; /* Documents in table */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ @@ -110907,47 +112049,70 @@ struct Fts3Cursor { #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ +} doclist; + /* ** A "phrase" is a sequence of one or more tokens that must match in ** sequence. A single token is the base case and the most common case. ** For a sequence of tokens contained in double-quotes (i.e. "one two three") ** nToken will be the number of tokens in the string. -** -** The nDocMatch and nMatch variables contain data that may be used by the -** matchinfo() function. They are populated when the full-text index is -** queried for hits on the phrase. If one or more tokens in the phrase -** are deferred, the nDocMatch and nMatch variables are populated based -** on the assumption that the */ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ - int bFulltext; /* True if full-text index was used */ - Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */ + + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ }; struct Fts3Phrase { - /* Variables populated by fts3_expr.c when parsing a MATCH expression */ + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ + int iDoclistToken; + + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ - int isNot; /* Phrase prefixed by unary not (-) operator */ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ }; /* ** A tree of these objects forms the RHS of a MATCH operator. ** -** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded -** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, -** containing the results of the NEAR or phrase query in FTS3 doclist -** format. As usual, the initial "Length" field found in doclists stored -** on disk is omitted from this buffer. +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. ** -** Variable pCurrent always points to the start of a docid field within -** aDoclist. Since the doclist is usually scanned in docid order, this can -** be used to accelerate seeking to the required docid within the doclist. +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. */ struct Fts3Expr { int eType; /* One of the FTSQUERY_XXX values defined below */ @@ -110957,12 +112122,13 @@ struct Fts3Expr { Fts3Expr *pRight; /* Right operand */ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ - int isLoaded; /* True if aDoclist/nDoclist are initialized. */ - char *aDoclist; /* Buffer containing doclist */ - int nDoclist; /* Size of aDoclist in bytes */ + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ - sqlite3_int64 iCurrent; - char *pCurrent; + u32 *aMI; }; /* @@ -110990,12 +112156,12 @@ SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); -SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); +SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( + Fts3Table*,int,const char*,int,int,Fts3SegReader**); SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); -SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); -SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); +SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); @@ -111004,17 +112170,18 @@ SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); -SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); -#define FTS3_SEGCURSOR_PENDING -1 -#define FTS3_SEGCURSOR_ALL -2 +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_ALL -2 + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); -SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); -SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); -SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( - Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *); + Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 @@ -111031,7 +112198,7 @@ struct Fts3SegFilter { int flags; }; -struct Fts3SegReaderCursor { +struct Fts3MultiSegReader { /* Used internally by sqlite3Fts3SegReaderXXX() calls */ Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ int nSegment; /* Size of apSegment array */ @@ -111040,8 +112207,12 @@ struct Fts3SegReaderCursor { char *aBuffer; /* Buffer to merge doclists in */ int nBuffer; /* Allocated size of aBuffer[] in bytes */ - /* Cost of running this iterator. Used by fts3.c only. */ - int nCost; + int iColFilter; /* If >=0, filter for this column */ + int bRestart; + + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ char *zTerm; /* Pointer to term buffer */ @@ -111056,11 +112227,9 @@ SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); -SQLITE_PRIVATE char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int); -SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); -SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); -SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); /* fts3_tokenizer.c */ SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); @@ -111084,15 +112253,45 @@ SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); +SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); #endif /* fts3_aux.c */ SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); +SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor( + Fts3Cursor *pCsr, /* Virtual table cursor handle */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ +); + +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + +SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); +SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr); + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); + +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); + +#endif /* SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ /************** End of fts3Int.h *********************************************/ /************** Continuing where we left off in fts3.c ***********************/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) +# define SQLITE_CORE 1 +#endif #ifndef SQLITE_CORE @@ -111206,17 +112405,31 @@ static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ } /* -** As long as *pp has not reached its end (pEnd), then do the same -** as fts3GetDeltaVarint(): read a single varint and add it to *pVal. -** But if we have reached the end of the varint, just set *pp=0 and -** leave *pVal unchanged. +** When this function is called, *pp points to the first byte following a +** varint that is part of a doclist (or position-list, or any other list +** of varints). This function moves *pp to point to the start of that varint, +** and sets *pVal by the varint value. +** +** Argument pStart points to the first byte of the doclist that the +** varint is part of. */ -static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){ - if( *pp>=pEnd ){ - *pp = 0; - }else{ - fts3GetDeltaVarint(pp, pVal); - } +static void fts3GetReverseVarint( + char **pp, + char *pStart, + sqlite3_int64 *pVal +){ + sqlite3_int64 iVal; + char *p = *pp; + + /* Pointer p now points at the first byte past the varint we are + ** interested in. So, unless the doclist is corrupt, the 0x80 bit is + ** clear on character p[-1]. */ + for(p = (*pp)-2; p>=pStart && *p&0x80; p--); + p++; + *pp = p; + + sqlite3Fts3GetVarint(p, &iVal); + *pVal = iVal; } /* @@ -111310,6 +112523,8 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ char *zSql; /* SQL statement passed to declare_vtab() */ char *zCols; /* List of user defined columns */ + sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + /* Create a list of user columns for the virtual table */ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); for(i=1; zCols && inColumn; i++){ @@ -111415,6 +112630,9 @@ static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ sqlite3_step(pStmt); p->nPgsz = sqlite3_column_int(pStmt, 0); rc = sqlite3_finalize(pStmt); + }else if( rc==SQLITE_AUTH ){ + p->nPgsz = 1024; + rc = SQLITE_OK; } } assert( p->nPgsz>0 || rc!=SQLITE_OK ); @@ -111588,6 +112806,58 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ return zRet; } +static int fts3GobbleInt(const char **pp, int *pnOut){ + const char *p = *pp; + int nInt = 0; + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ + nInt = nInt * 10 + (p[0] - '0'); + } + if( p==*pp ) return SQLITE_ERROR; + *pnOut = nInt; + *pp = p; + return SQLITE_OK; +} + + +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */ + struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */ +){ + struct Fts3Index *aIndex; + int nIndex = 1; + + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } + + aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); + *apIndex = *apFree = aIndex; + *pnIndex = nIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } + + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; i MATCHINFO */ + { "prefix", 6, 0 }, /* 1 -> PREFIX */ + { "compress", 8, 0 }, /* 2 -> COMPRESS */ + { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */ + { "order", 5, 0 } /* 4 -> ORDER */ + }; + + int iOpt; if( !zVal ){ rc = SQLITE_NOMEM; - goto fts3_init_out; - } - if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ - if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ - bNoDocsize = 1; - }else{ - *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); - rc = SQLITE_ERROR; - } - }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){ - zCompress = zVal; - zVal = 0; - }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ - zUncompress = zVal; - zVal = 0; }else{ - *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); - rc = SQLITE_ERROR; + for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ + break; + } + } + if( iOpt==SizeofArray(aFts4Opt) ){ + *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + }else{ + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; + + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; + + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; + + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) + ){ + *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + } + } + sqlite3_free(zVal); } - sqlite3_free(zVal); } /* Otherwise, the argument is a column name. */ @@ -111711,10 +113032,17 @@ static int fts3InitVtab( } assert( pTokenizer ); + rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree); + if( rc==SQLITE_ERROR ){ + assert( zPrefix ); + *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); + } + if( rc!=SQLITE_OK ) goto fts3_init_out; /* Allocate and populate the Fts3Table structure. */ - nByte = sizeof(Fts3Table) + /* Fts3Table */ + nByte = sizeof(Fts3Table) + /* Fts3Table */ nCol * sizeof(char *) + /* azColumn */ + nIndex * sizeof(struct Fts3Index) + /* aIndex */ nName + /* zName */ nDb + /* zDb */ nString; /* Space for azColumn strings */ @@ -111729,14 +113057,22 @@ static int fts3InitVtab( p->nPendingData = 0; p->azColumn = (char **)&p[1]; p->pTokenizer = pTokenizer; - p->nNodeSize = 1000; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; - fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); + p->bDescIdx = bDescIdx; + TESTONLY( p->inTransaction = -1 ); + TESTONLY( p->mxSavepoint = -1 ); + + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } /* Fill in the zName and zDb fields of the vtab structure. */ - zCsr = (char *)&p->azColumn[nCol]; + zCsr = (char *)&p->aIndex[nIndex]; p->zName = zCsr; memcpy(zCsr, argv[2], nName); zCsr += nName; @@ -111747,7 +113083,7 @@ static int fts3InitVtab( /* Fill in the azColumn array */ for(iCol=0; iColnNodeSize = p->nPgsz-35; /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aFree); sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free((void *)aCol); @@ -111793,6 +113130,7 @@ fts3_init_out: pTokenizer->pModule->xDestroy(pTokenizer); } }else{ + assert( p->pSegments==0 ); *ppVTab = &p->base; } return rc; @@ -111878,6 +113216,23 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ pInfo->aConstraintUsage[iCons].argvIndex = 1; pInfo->aConstraintUsage[iCons].omit = 1; } + + /* Regardless of the strategy selected, FTS can deliver rows in rowid (or + ** docid) order. Both ascending and descending are possible. + */ + if( pInfo->nOrderBy==1 ){ + struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc ){ + pInfo->idxStr = "DESC"; + }else{ + pInfo->idxStr = "ASC"; + } + pInfo->orderByConsumed = 1; + } + } + + assert( p->pSegments==0 ); return SQLITE_OK; } @@ -111913,6 +113268,7 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); sqlite3_free(pCsr->aMatchinfo); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_free(pCsr); return SQLITE_OK; } @@ -111924,8 +113280,8 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ if( pCsr->isRequireSeek ){ - pCsr->isRequireSeek = 0; sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ return SQLITE_OK; }else{ @@ -111935,7 +113291,7 @@ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ ** table is missing a row that is present in the full-text index. ** The data structures are corrupt. */ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } pCsr->isEof = 1; if( pContext ){ @@ -111995,7 +113351,7 @@ static int fts3ScanInteriorNode( zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); if( zCsr>zEnd ){ - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } while( zCsrzEnd ){ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; goto finish_scan; } if( nPrefix+nSuffix>nAlloc ){ @@ -112106,7 +113462,7 @@ static int fts3SelectLeaf( int nBlob; /* Size of zBlob in bytes */ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ - rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob); + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); } @@ -112116,7 +113472,7 @@ static int fts3SelectLeaf( } if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob); + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); } if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); @@ -112492,7 +113848,19 @@ static int fts3PoslistPhraseMerge( } /* -** Merge two position-lists as required by the NEAR operator. +** Merge two position-lists as required by the NEAR operator. The argument +** position lists correspond to the left and right phrases of an expression +** like: +** +** "phrase 1" NEAR "phrase number 2" +** +** Position list *pp1 corresponds to the left-hand side of the NEAR +** expression and *pp2 to the right. As usual, the indexes in the position +** lists are the offsets of the last token in each phrase (tokens "1" and "2" +** in the example above). +** +** The output position list - written to *pp - is a copy of *pp2 with those +** entries that are not sufficiently NEAR entries in *pp1 removed. */ static int fts3PoslistNearMerge( char **pp, /* Output buffer */ @@ -112505,214 +113873,27 @@ static int fts3PoslistNearMerge( char *p1 = *pp1; char *p2 = *pp2; - if( !pp ){ - if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1; - *pp1 = p1; - *pp2 = p2; - return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1); + char *pTmp1 = aTmp; + char *pTmp2; + char *aTmp2; + int res = 1; + + fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); + aTmp2 = pTmp2 = pTmp1; + *pp1 = p1; + *pp2 = p2; + fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); + if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ + fts3PoslistMerge(pp, &aTmp, &aTmp2); + }else if( pTmp1!=aTmp ){ + fts3PoslistCopy(pp, &aTmp); + }else if( pTmp2!=aTmp2 ){ + fts3PoslistCopy(pp, &aTmp2); }else{ - char *pTmp1 = aTmp; - char *pTmp2; - char *aTmp2; - int res = 1; - - fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); - aTmp2 = pTmp2 = pTmp1; - *pp1 = p1; - *pp2 = p2; - fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); - if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ - fts3PoslistMerge(pp, &aTmp, &aTmp2); - }else if( pTmp1!=aTmp ){ - fts3PoslistCopy(pp, &aTmp); - }else if( pTmp2!=aTmp2 ){ - fts3PoslistCopy(pp, &aTmp2); - }else{ - res = 0; - } - - return res; - } -} - -/* -** Values that may be used as the first parameter to fts3DoclistMerge(). -*/ -#define MERGE_NOT 2 /* D + D -> D */ -#define MERGE_AND 3 /* D + D -> D */ -#define MERGE_OR 4 /* D + D -> D */ -#define MERGE_POS_OR 5 /* P + P -> P */ -#define MERGE_PHRASE 6 /* P + P -> D */ -#define MERGE_POS_PHRASE 7 /* P + P -> P */ -#define MERGE_NEAR 8 /* P + P -> D */ -#define MERGE_POS_NEAR 9 /* P + P -> P */ - -/* -** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2 -** (size n2 bytes). The output is written to pre-allocated buffer aBuffer, -** which is guaranteed to be large enough to hold the results. The number -** of bytes written to aBuffer is stored in *pnBuffer before returning. -** -** If successful, SQLITE_OK is returned. Otherwise, if a malloc error -** occurs while allocating a temporary buffer as part of the merge operation, -** SQLITE_NOMEM is returned. -*/ -static int fts3DoclistMerge( - int mergetype, /* One of the MERGE_XXX constants */ - int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ - int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ - char *aBuffer, /* Pre-allocated output buffer */ - int *pnBuffer, /* OUT: Bytes written to aBuffer */ - char *a1, /* Buffer containing first doclist */ - int n1, /* Size of buffer a1 */ - char *a2, /* Buffer containing second doclist */ - int n2, /* Size of buffer a2 */ - int *pnDoc /* OUT: Number of docids in output */ -){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - - char *p = aBuffer; - char *p1 = a1; - char *p2 = a2; - char *pEnd1 = &a1[n1]; - char *pEnd2 = &a2[n2]; - int nDoc = 0; - - assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR - || mergetype==MERGE_AND || mergetype==MERGE_NOT - || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE - || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR - ); - - if( !aBuffer ){ - *pnBuffer = 0; - return SQLITE_NOMEM; + res = 0; } - /* Read the first docid from each doclist */ - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - - switch( mergetype ){ - case MERGE_OR: - case MERGE_POS_OR: - while( p1 || p2 ){ - if( p2 && p1 && i1==i2 ){ - fts3PutDeltaVarint(&p, &iPrev, i1); - if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - }else if( !p2 || (p1 && i1=pEnd ){ + *pp = 0; + }else{ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + if( bDescIdx ){ + *pVal -= iVal; + }else{ + *pVal += iVal; + } + } +} + +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_int64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + iWrite = iVal - *piPrev; + }else{ + iWrite = *piPrev - iVal; + } + assert( *pbFirst || *piPrev==0 ); + assert( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; +} + +#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2)) + +static int fts3DoclistOrMerge( + int bDescIdx, /* True if arguments are desc */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + char *aOut; + int bFirstOut = 0; + + *paOut = 0; + *pnOut = 0; + aOut = sqlite3_malloc(n1+n2); + if( !aOut ) return SQLITE_NOMEM; + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistMerge(&p, &p1, &p2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + *paOut = aOut; + *pnOut = (p-aOut); + return SQLITE_OK; +} + +static void fts3DoclistPhraseMerge( + int bDescIdx, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + char *aLeft, int nLeft, /* Left doclist */ + char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + int bFirstOut = 0; + char *aOut = aRight; + + assert( nDist>0 ); + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + *pnRight = p - aOut; +} + + /* ** Merge all doclists in the TermSelect.aaOutput[] array into a single ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all @@ -112735,8 +114058,7 @@ struct TermSelect { ** the responsibility of the caller to free any doclists left in the ** TermSelect.aaOutput[] array. */ -static int fts3TermSelectMerge(TermSelect *pTS){ - int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); +static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ char *aOut = 0; int nOut = 0; int i; @@ -112751,15 +114073,17 @@ static int fts3TermSelectMerge(TermSelect *pTS){ nOut = pTS->anOutput[i]; pTS->aaOutput[i] = 0; }else{ - int nNew = nOut + pTS->anOutput[i]; - char *aNew = sqlite3_malloc(nNew); - if( !aNew ){ - sqlite3_free(aOut); - return SQLITE_NOMEM; - } - fts3DoclistMerge(mergetype, 0, 0, - aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0 + int nNew; + char *aNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + return rc; + } + sqlite3_free(pTS->aaOutput[i]); sqlite3_free(aOut); pTS->aaOutput[i] = 0; @@ -112795,9 +114119,7 @@ static int fts3TermSelectCb( if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). TODO: Add a way to transfer control of the - ** aDoclist buffer from the caller so as to avoid the memcpy(). - */ + ** buffer using memcpy(). */ pTS->aaOutput[0] = sqlite3_malloc(nDoclist); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ @@ -112806,126 +114128,100 @@ static int fts3TermSelectCb( return SQLITE_NOMEM; } }else{ - int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); char *aMerge = aDoclist; int nMerge = nDoclist; int iOut; for(iOut=0; iOutaaOutput); iOut++){ - char *aNew; - int nNew; if( pTS->aaOutput[iOut]==0 ){ assert( iOut>0 ); pTS->aaOutput[iOut] = aMerge; pTS->anOutput[iOut] = nMerge; break; - } + }else{ + char *aNew; + int nNew; - nNew = nMerge + pTS->anOutput[iOut]; - aNew = sqlite3_malloc(nNew); - if( !aNew ){ - if( aMerge!=aDoclist ){ - sqlite3_free(aMerge); + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; } - return SQLITE_NOMEM; - } - fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, - pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0 - ); - if( iOut>0 ) sqlite3_free(aMerge); - sqlite3_free(pTS->aaOutput[iOut]); - pTS->aaOutput[iOut] = 0; - - aMerge = aNew; - nMerge = nNew; - if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } } } } return SQLITE_OK; } -static int fts3DeferredTermSelect( - Fts3DeferredToken *pToken, /* Phrase token */ - int isTermPos, /* True to include positions */ - int *pnOut, /* OUT: Size of list */ - char **ppOut /* OUT: Body of list */ +/* +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +*/ +static int fts3SegReaderCursorAppend( + Fts3MultiSegReader *pCsr, + Fts3SegReader *pNew ){ - char *aSource; - int nSource; - - aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource); - if( !aSource ){ - *pnOut = 0; - *ppOut = 0; - }else if( isTermPos ){ - *ppOut = sqlite3_malloc(nSource); - if( !*ppOut ) return SQLITE_NOMEM; - memcpy(*ppOut, aSource, nSource); - *pnOut = nSource; - }else{ - sqlite3_int64 docid; - *pnOut = sqlite3Fts3GetVarint(aSource, &docid); - *ppOut = sqlite3_malloc(*pnOut); - if( !*ppOut ) return SQLITE_NOMEM; - sqlite3Fts3PutVarint(*ppOut, docid); + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); + if( !apNew ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; + } + pCsr->apSegment = apNew; } - + pCsr->apSegment[pCsr->nSegment++] = pNew; return SQLITE_OK; } -SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( +static int fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ - Fts3SegReaderCursor *pCsr /* Cursor object to populate */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ int rc = SQLITE_OK; int rc2; - int iAge = 0; sqlite3_stmt *pStmt = 0; - Fts3SegReader *pPending = 0; - assert( iLevel==FTS3_SEGCURSOR_ALL - || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel>=0 - ); - assert( FTS3_SEGCURSOR_PENDING<0 ); - assert( FTS3_SEGCURSOR_ALL<0 ); - assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) ); - assert( isPrefix==0 || isScan==0 ); - - - memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); - - /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ - assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 ); - if( iLevel<0 && isScan==0 ){ - rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending); - if( rc==SQLITE_OK && pPending ){ - int nByte = (sizeof(Fts3SegReader *) * 16); - pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); - if( pCsr->apSegment==0 ){ - rc = SQLITE_NOMEM; - }else{ - pCsr->apSegment[0] = pPending; - pCsr->nSegment = 1; - pPending = 0; - } + /* If iLevel is less than 0 and this is not a scan, include a seg-reader + ** for the pending-terms. If this is a scan, then this call must be being + ** made by an fts4aux module, not an FTS table. In this case calling + ** Fts3SegReaderPending might segfault, as the data structures used by + ** fts4aux are not completely populated. So it's easiest to filter these + ** calls out here. */ + if( iLevel<0 && p->aIndex ){ + Fts3SegReader *pSeg = 0; + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg); + if( rc==SQLITE_OK && pSeg ){ + rc = fts3SegReaderCursorAppend(pCsr, pSeg); } } if( iLevel!=FTS3_SEGCURSOR_PENDING ){ if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt); + rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt); } + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pSeg = 0; /* Read the values returned by the SELECT into local variables. */ sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); @@ -112934,18 +114230,6 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( int nRoot = sqlite3_column_bytes(pStmt, 4); char const *zRoot = sqlite3_column_blob(pStmt, 4); - /* If nSegment is a multiple of 16 the array needs to be extended. */ - if( (pCsr->nSegment%16)==0 ){ - Fts3SegReader **apNew; - int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); - apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); - if( !apNew ){ - rc = SQLITE_NOMEM; - goto finished; - } - pCsr->apSegment = apNew; - } - /* If zTerm is not NULL, and this segment is not stored entirely on its ** root node, the range of leaves scanned can be reduced. Do this. */ if( iStartBlock && zTerm ){ @@ -112955,53 +114239,117 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; } - rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock, - iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment] + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg ); if( rc!=SQLITE_OK ) goto finished; - pCsr->nSegment++; - iAge++; + rc = fts3SegReaderCursorAppend(pCsr, pSeg); } } finished: rc2 = sqlite3_reset(pStmt); if( rc==SQLITE_DONE ) rc = rc2; - sqlite3Fts3SegReaderFree(pPending); return rc; } +/* +** Set up a cursor object for iterating through a full-text index or a +** single level therein. +*/ +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevelaIndex==0 ); + + memset(pCsr, 0, sizeof(Fts3MultiSegReader)); + + return fts3SegReaderCursor( + p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr + ); +} + +static int fts3SegReaderCursorAddZero( + Fts3Table *p, + const char *zTerm, + int nTerm, + Fts3MultiSegReader *pCsr +){ + return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); +} + + +SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ - Fts3SegReaderCursor **ppSegcsr /* OUT: Allocated seg-reader cursor */ + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ){ - Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */ + Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ - pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); + pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); if( pSegcsr ){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int i; - int nCost = 0; - rc = sqlite3Fts3SegReaderCursor( - p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr); - - for(i=0; rc==SQLITE_OK && inSegment; i++){ - rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); + int bFound = 0; /* True once an index has been found */ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + + if( isPrefix ){ + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor( + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); + pSegcsr->bLookup = 1; + } + } + + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm+1 ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor( + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + ); + if( rc==SQLITE_OK ){ + rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr); + } + } + } + } + + if( bFound==0 ){ + rc = sqlite3Fts3SegReaderCursor( + p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + ); + pSegcsr->bLookup = !isPrefix; } - pSegcsr->nCost = nCost; } *ppSegcsr = pSegcsr; return rc; } -static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){ +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ sqlite3Fts3SegReaderFinish(pSegcsr); sqlite3_free(pSegcsr); } @@ -113026,7 +114374,7 @@ static int fts3TermSelect( char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ - Fts3SegReaderCursor *pSegcsr; /* Seg-reader cursor for this term */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ TermSelect tsc; /* Context object for fts3TermSelectCb() */ Fts3SegFilter filter; /* Segment term filter configuration */ @@ -113052,7 +114400,7 @@ static int fts3TermSelect( } if( rc==SQLITE_OK ){ - rc = fts3TermSelectMerge(&tsc); + rc = fts3TermSelectMerge(p, &tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; @@ -113102,660 +114450,6 @@ static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ return nDoc; } -/* -** Call sqlite3Fts3DeferToken() for each token in the expression pExpr. -*/ -static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc = SQLITE_OK; - if( pExpr ){ - rc = fts3DeferExpression(pCsr, pExpr->pLeft); - if( rc==SQLITE_OK ){ - rc = fts3DeferExpression(pCsr, pExpr->pRight); - } - if( pExpr->eType==FTSQUERY_PHRASE ){ - int iCol = pExpr->pPhrase->iColumn; - int i; - for(i=0; rc==SQLITE_OK && ipPhrase->nToken; i++){ - Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; - if( pToken->pDeferred==0 ){ - rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol); - } - } - } - } - return rc; -} - -/* -** This function removes the position information from a doclist. When -** called, buffer aList (size *pnList bytes) contains a doclist that includes -** position information. This function removes the position information so -** that aList contains only docids, and adjusts *pnList to reflect the new -** (possibly reduced) size of the doclist. -*/ -static void fts3DoclistStripPositions( - char *aList, /* IN/OUT: Buffer containing doclist */ - int *pnList /* IN/OUT: Size of doclist in bytes */ -){ - if( aList ){ - char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */ - char *p = aList; /* Input cursor */ - char *pOut = aList; /* Output cursor */ - - while( piColumn; - int isTermPos = (pPhrase->nToken>1 || isReqPos); - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - int isFirst = 1; - - int iPrevTok = 0; - int nDoc = 0; - - /* If this is an xFilter() evaluation, create a segment-reader for each - ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo - ** evaluation, only create segment-readers if there are no Fts3DeferredToken - ** objects attached to the phrase-tokens. - */ - for(ii=0; iinToken; ii++){ - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pSegcsr==0 ){ - if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) - || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) - || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) - ){ - rc = fts3TermSegReaderCursor( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr - ); - if( rc!=SQLITE_OK ) return rc; - } - } - } - - for(ii=0; iinToken; ii++){ - Fts3PhraseToken *pTok; /* Token to find doclist for */ - int iTok = 0; /* The token being queried this iteration */ - char *pList = 0; /* Pointer to token doclist */ - int nList = 0; /* Size of buffer at pList */ - - /* Select a token to process. If this is an xFilter() call, then tokens - ** are processed in order from least to most costly. Otherwise, tokens - ** are processed in the order in which they occur in the phrase. - */ - if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){ - assert( isReqPos ); - iTok = ii; - pTok = &pPhrase->aToken[iTok]; - if( pTok->bFulltext==0 ) continue; - }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){ - iTok = ii; - pTok = &pPhrase->aToken[iTok]; - }else{ - int nMinCost = 0x7FFFFFFF; - int jj; - - /* Find the remaining token with the lowest cost. */ - for(jj=0; jjnToken; jj++){ - Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr; - if( pSegcsr && pSegcsr->nCostnCost; - } - } - pTok = &pPhrase->aToken[iTok]; - - /* This branch is taken if it is determined that loading the doclist - ** for the next token would require more IO than loading all documents - ** currently identified by doclist pOut/nOut. No further doclists will - ** be loaded from the full-text index for this phrase. - */ - if( nMinCost>nDoc && ii>0 ){ - rc = fts3DeferExpression(pCsr, pCsr->pExpr); - break; - } - } - - if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ - rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); - }else{ - if( pTok->pSegcsr ){ - rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); - } - pTok->bFulltext = 1; - } - assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 ); - if( rc!=SQLITE_OK ) break; - - if( isFirst ){ - pOut = pList; - nOut = nList; - if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){ - nDoc = fts3DoclistCountDocids(1, pOut, nOut); - } - isFirst = 0; - iPrevTok = iTok; - }else{ - /* Merge the new term list and the current output. */ - char *aLeft, *aRight; - int nLeft, nRight; - int nDist; - int mt; - - /* If this is the final token of the phrase, and positions were not - ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE. - ** This drops the position information from the output list. - */ - mt = MERGE_POS_PHRASE; - if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE; - - assert( iPrevTok!=iTok ); - if( iPrevToknToken ){ - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 ); - fts3DoclistStripPositions(pOut, &nOut); - } - *paOut = pOut; - *pnOut = nOut; - }else{ - sqlite3_free(pOut); - } - return rc; -} - -/* -** This function merges two doclists according to the requirements of a -** NEAR operator. -** -** Both input doclists must include position information. The output doclist -** includes position information if the first argument to this function -** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR. -*/ -static int fts3NearMerge( - int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ - int nNear, /* Parameter to NEAR operator */ - int nTokenLeft, /* Number of tokens in LHS phrase arg */ - char *aLeft, /* Doclist for LHS (incl. positions) */ - int nLeft, /* Size of LHS doclist in bytes */ - int nTokenRight, /* As nTokenLeft */ - char *aRight, /* As aLeft */ - int nRight, /* As nRight */ - char **paOut, /* OUT: Results of merge (malloced) */ - int *pnOut /* OUT: Sized of output buffer */ -){ - char *aOut; /* Buffer to write output doclist to */ - int rc; /* Return code */ - - assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR ); - - aOut = sqlite3_malloc(nLeft+nRight+1); - if( aOut==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, - aOut, pnOut, aLeft, nLeft, aRight, nRight, 0 - ); - if( rc!=SQLITE_OK ){ - sqlite3_free(aOut); - aOut = 0; - } - } - - *paOut = aOut; - return rc; -} - -/* -** This function is used as part of the processing for the snippet() and -** offsets() functions. -** -** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both -** have their respective doclists (including position information) loaded -** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from -** each doclist that are not within nNear tokens of a corresponding entry -** in the other doclist. -*/ -SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ - int rc; /* Return code */ - - assert( pLeft->eType==FTSQUERY_PHRASE ); - assert( pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->isLoaded && pRight->isLoaded ); - - if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ - sqlite3_free(pLeft->aDoclist); - sqlite3_free(pRight->aDoclist); - pRight->aDoclist = 0; - pLeft->aDoclist = 0; - rc = SQLITE_OK; - }else{ - char *aOut; /* Buffer in which to assemble new doclist */ - int nOut; /* Size of buffer aOut in bytes */ - - rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, - &aOut, &nOut - ); - if( rc!=SQLITE_OK ) return rc; - sqlite3_free(pRight->aDoclist); - pRight->aDoclist = aOut; - pRight->nDoclist = nOut; - - rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, - pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, - &aOut, &nOut - ); - sqlite3_free(pLeft->aDoclist); - pLeft->aDoclist = aOut; - pLeft->nDoclist = nOut; - } - return rc; -} - - -/* -** Allocate an Fts3SegReaderArray for each token in the expression pExpr. -** The allocated objects are stored in the Fts3PhraseToken.pArray member -** variables of each token structure. -*/ -static int fts3ExprAllocateSegReaders( - Fts3Cursor *pCsr, /* FTS3 table */ - Fts3Expr *pExpr, /* Expression to create seg-readers for */ - int *pnExpr /* OUT: Number of AND'd expressions */ -){ - int rc = SQLITE_OK; /* Return code */ - - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER ); - if( pnExpr && pExpr->eType!=FTSQUERY_AND ){ - (*pnExpr)++; - pnExpr = 0; - } - - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - - for(ii=0; rc==SQLITE_OK && iinToken; ii++){ - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pSegcsr==0 ){ - rc = fts3TermSegReaderCursor( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr - ); - } - } - }else{ - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); - if( rc==SQLITE_OK ){ - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr); - } - } - return rc; -} - -/* -** Free the Fts3SegReaderArray objects associated with each token in the -** expression pExpr. In other words, this function frees the resources -** allocated by fts3ExprAllocateSegReaders(). -*/ -static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ - if( pExpr ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - if( pPhrase ){ - int kk; - for(kk=0; kknToken; kk++){ - fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr); - pPhrase->aToken[kk].pSegcsr = 0; - } - } - fts3ExprFreeSegReaders(pExpr->pLeft); - fts3ExprFreeSegReaders(pExpr->pRight); - } -} - -/* -** Return the sum of the costs of all tokens in the expression pExpr. This -** function must be called after Fts3SegReaderArrays have been allocated -** for all tokens using fts3ExprAllocateSegReaders(). -*/ -static int fts3ExprCost(Fts3Expr *pExpr){ - int nCost; /* Return value */ - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - nCost = 0; - for(ii=0; iinToken; ii++){ - Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr; - if( pSegcsr ) nCost += pSegcsr->nCost; - } - }else{ - nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); - } - return nCost; -} - -/* -** The following is a helper function (and type) for fts3EvalExpr(). It -** must be called after Fts3SegReaders have been allocated for every token -** in the expression. See the context it is called from in fts3EvalExpr() -** for further explanation. -*/ -typedef struct ExprAndCost ExprAndCost; -struct ExprAndCost { - Fts3Expr *pExpr; - int nCost; -}; -static void fts3ExprAssignCosts( - Fts3Expr *pExpr, /* Expression to create seg-readers for */ - ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */ -){ - if( pExpr->eType==FTSQUERY_AND ){ - fts3ExprAssignCosts(pExpr->pLeft, ppExprCost); - fts3ExprAssignCosts(pExpr->pRight, ppExprCost); - }else{ - (*ppExprCost)->pExpr = pExpr; - (*ppExprCost)->nCost = fts3ExprCost(pExpr); - (*ppExprCost)++; - } -} - -/* -** Evaluate the full-text expression pExpr against FTS3 table pTab. Store -** the resulting doclist in *paOut and *pnOut. This routine mallocs for -** the space needed to store the output. The caller is responsible for -** freeing the space when it has finished. -** -** This function is called in two distinct contexts: -** -** * From within the virtual table xFilter() method. In this case, the -** output doclist contains entries for all rows in the table, based on -** data read from the full-text index. -** -** In this case, if the query expression contains one or more tokens that -** are very common, then the returned doclist may contain a superset of -** the documents that actually match the expression. -** -** * From within the virtual table xNext() method. This call is only made -** if the call from within xFilter() found that there were very common -** tokens in the query expression and did return a superset of the -** matching documents. In this case the returned doclist contains only -** entries that correspond to the current row of the table. Instead of -** reading the data for each token from the full-text index, the data is -** already available in-memory in the Fts3PhraseToken.pDeferred structures. -** See fts3EvalDeferred() for how it gets there. -** -** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is -** required) Fts3Cursor.doDeferred==1. -** -** If the SQLite invokes the snippet(), offsets() or matchinfo() function -** as part of a SELECT on an FTS3 table, this function is called on each -** individual phrase expression in the query. If there were very common tokens -** found in the xFilter() call, then this function is called once for phrase -** for each row visited, and the returned doclist contains entries for the -** current row only. Otherwise, if there were no very common tokens, then this -** function is called once only for each phrase in the query and the returned -** doclist contains entries for all rows of the table. -** -** Fts3Cursor.doDeferred==1 when this function is called on phrases as a -** result of a snippet(), offsets() or matchinfo() invocation. -*/ -static int fts3EvalExpr( - Fts3Cursor *p, /* Virtual table cursor handle */ - Fts3Expr *pExpr, /* Parsed fts3 expression */ - char **paOut, /* OUT: Pointer to malloc'd result buffer */ - int *pnOut, /* OUT: Size of buffer at *paOut */ - int isReqPos /* Require positions in output buffer */ -){ - int rc = SQLITE_OK; /* Return code */ - - /* Zero the output parameters. */ - *paOut = 0; - *pnOut = 0; - - if( pExpr ){ - assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT - || pExpr->eType==FTSQUERY_PHRASE - ); - assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 ); - - if( pExpr->eType==FTSQUERY_PHRASE ){ - rc = fts3PhraseSelect(p, pExpr->pPhrase, - isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR), - paOut, pnOut - ); - fts3ExprFreeSegReaders(pExpr); - }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){ - ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */ - int nExpr = 0; /* Size of aExpr[] */ - char *aRet = 0; /* Doclist to return to caller */ - int nRet = 0; /* Length of aRet[] in bytes */ - int nDoc = 0x7FFFFFFF; - - assert( !isReqPos ); - - rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr); - if( rc==SQLITE_OK ){ - assert( nExpr>1 ); - aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr); - if( !aExpr ) rc = SQLITE_NOMEM; - } - if( rc==SQLITE_OK ){ - int ii; /* Used to iterate through expressions */ - - fts3ExprAssignCosts(pExpr, &aExpr); - aExpr -= nExpr; - for(ii=0; iipExpr && (pBest==0 || pCand->nCostnCost) ){ - pBest = pCand; - } - } - - if( pBest->nCost>nDoc ){ - rc = fts3DeferExpression(p, p->pExpr); - break; - }else{ - rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0); - if( rc!=SQLITE_OK ) break; - pBest->pExpr = 0; - if( ii==0 ){ - aRet = aNew; - nRet = nNew; - nDoc = fts3DoclistCountDocids(0, aRet, nRet); - }else{ - fts3DoclistMerge( - MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc - ); - sqlite3_free(aNew); - } - } - } - } - - if( rc==SQLITE_OK ){ - *paOut = aRet; - *pnOut = nRet; - }else{ - assert( *paOut==0 ); - sqlite3_free(aRet); - } - sqlite3_free(aExpr); - fts3ExprFreeSegReaders(pExpr); - - }else{ - char *aLeft; - char *aRight; - int nLeft; - int nRight; - - assert( pExpr->eType==FTSQUERY_NEAR - || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_NOT - || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT) - ); - - if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) - && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) - ){ - switch( pExpr->eType ){ - case FTSQUERY_NEAR: { - Fts3Expr *pLeft; - Fts3Expr *pRight; - int mergetype = MERGE_NEAR; - if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ - mergetype = MERGE_POS_NEAR; - } - pLeft = pExpr->pLeft; - while( pLeft->eType==FTSQUERY_NEAR ){ - pLeft=pLeft->pRight; - } - pRight = pExpr->pRight; - assert( pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->eType==FTSQUERY_PHRASE ); - - rc = fts3NearMerge(mergetype, pExpr->nNear, - pLeft->pPhrase->nToken, aLeft, nLeft, - pRight->pPhrase->nToken, aRight, nRight, - paOut, pnOut - ); - sqlite3_free(aLeft); - break; - } - - case FTSQUERY_OR: { - /* Allocate a buffer for the output. The maximum size is the - ** sum of the sizes of the two input buffers. The +1 term is - ** so that a buffer of zero bytes is never allocated - this can - ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. - */ - char *aBuffer = sqlite3_malloc(nRight+nLeft+1); - rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, - aLeft, nLeft, aRight, nRight, 0 - ); - *paOut = aBuffer; - sqlite3_free(aLeft); - break; - } - - default: { - assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); - fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, - aLeft, nLeft, aRight, nRight, 0 - ); - *paOut = aLeft; - break; - } - } - } - sqlite3_free(aRight); - } - } - - assert( rc==SQLITE_OK || *paOut==0 ); - return rc; -} - -/* -** This function is called from within xNext() for each row visited by -** an FTS3 query. If evaluating the FTS3 query expression within xFilter() -** was able to determine the exact set of matching rows, this function sets -** *pbRes to true and returns SQLITE_IO immediately. -** -** Otherwise, if evaluating the query expression within xFilter() returned a -** superset of the matching documents instead of an exact set (this happens -** when the query includes very common tokens and it is deemed too expensive to -** load their doclists from disk), this function tests if the current row -** really does match the FTS3 query. -** -** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK -** is returned and *pbRes is set to true if the current row matches the -** FTS3 query (and should be included in the results returned to SQLite), or -** false otherwise. -*/ -static int fts3EvalDeferred( - Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */ - int *pbRes /* OUT: Set to true if row is a match */ -){ - int rc = SQLITE_OK; - if( pCsr->pDeferred==0 ){ - *pbRes = 1; - }else{ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - sqlite3Fts3FreeDeferredDoclists(pCsr); - rc = sqlite3Fts3CacheDeferredDoclists(pCsr); - } - if( rc==SQLITE_OK ){ - char *a = 0; - int n = 0; - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0); - assert( n>=0 ); - *pbRes = (n>0); - sqlite3_free(a); - } - } - return rc; -} - /* ** Advance the cursor to the next row in the %_content table that ** matches the search criteria. For a MATCH search, this will be @@ -113768,31 +114462,20 @@ static int fts3EvalDeferred( ** subsequently to determine whether or not an EOF was hit. */ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ - int res; - int rc = SQLITE_OK; /* Return code */ + int rc; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - - pCsr->eEvalmode = FTS3_EVAL_NEXT; - do { - if( pCsr->aDoclist==0 ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - break; - } - pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); }else{ - if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ - pCsr->isEof = 1; - break; - } - sqlite3_reset(pCsr->pStmt); - fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + rc = SQLITE_OK; } - }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); - + }else{ + rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor); + } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -113819,11 +114502,7 @@ static int fts3FilterMethod( int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ - const char *azSql[] = { - "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */ - "SELECT %s FROM %Q.'%q_content' AS x ", /* full-scan */ - }; - int rc; /* Return code */ + int rc; char *zSql; /* SQL statement used to access %_content */ Fts3Table *p = (Fts3Table *)pCursor->pVtab; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; @@ -113842,6 +114521,13 @@ static int fts3FilterMethod( sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; + } + pCsr->eSearch = (i16)idxNum; + if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ int iCol = idxNum-FTS3_FULLTEXT_SEARCH; const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); @@ -113855,8 +114541,8 @@ static int fts3FilterMethod( ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ - p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]", - zQuery); + static const char *zErr = "malformed MATCH expression: [%s]"; + p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery); } return rc; } @@ -113864,7 +114550,8 @@ static int fts3FilterMethod( rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); + rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1); + sqlite3Fts3SegmentsClose(p); if( rc!=SQLITE_OK ) return rc; pCsr->pNextId = pCsr->aDoclist; @@ -113876,20 +114563,24 @@ static int fts3FilterMethod( ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ - zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH]; - zSql = sqlite3_mprintf(zSql, p->zReadExprlist, p->zDb, p->zName); - if( !zSql ){ - rc = SQLITE_NOMEM; + if( idxNum==FTS3_FULLSCAN_SEARCH ){ + const char *zSort = (pCsr->bDesc ? "DESC" : "ASC"); + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?"; + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName); } - if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); - } - pCsr->eSearch = (i16)idxNum; - + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); if( rc!=SQLITE_OK ) return rc; + + if( idxNum==FTS3_DOCID_SEARCH ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + if( rc!=SQLITE_OK ) return rc; + } + return fts3NextMethod(pCursor); } @@ -113909,16 +114600,7 @@ static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - if( pCsr->aDoclist ){ - *pRowid = pCsr->iPrevId; - }else{ - /* This branch runs if the query is implemented using a full-table scan - ** (not using the full-text index). In this case grab the rowid from the - ** SELECT statement. - */ - assert( pCsr->isRequireSeek==0 ); - *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); - } + *pRowid = pCsr->iPrevId; return SQLITE_OK; } @@ -113931,7 +114613,7 @@ static int fts3ColumnMethod( sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; Fts3Table *p = (Fts3Table *)pCursor->pVtab; @@ -113942,21 +114624,20 @@ static int fts3ColumnMethod( /* This call is a request for the "docid" column. Since "docid" is an ** alias for "rowid", use the xRowid() method to obtain the value. */ - sqlite3_int64 iRowid; - rc = fts3RowidMethod(pCursor, &iRowid); - sqlite3_result_int64(pContext, iRowid); + sqlite3_result_int64(pContext, pCsr->iPrevId); }else if( iCol==p->nColumn ){ /* The extra column whose name is the same as the table. ** Return a blob which is a pointer to the cursor. */ sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); - rc = SQLITE_OK; }else{ rc = fts3CursorSeek(0, pCsr); if( rc==SQLITE_OK ){ sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); } } + + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -113988,8 +114669,13 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ ** Implementation of xBegin() method. This is a no-op. */ static int fts3BeginMethod(sqlite3_vtab *pVtab){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); UNUSED_PARAMETER(pVtab); - assert( ((Fts3Table *)pVtab)->nPendingData==0 ); + assert( p->pSegments==0 ); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=1 ); + TESTONLY( p->inTransaction = 1 ); + TESTONLY( p->mxSavepoint = -1; ); return SQLITE_OK; } @@ -113999,8 +114685,13 @@ static int fts3BeginMethod(sqlite3_vtab *pVtab){ ** by fts3SyncMethod(). */ static int fts3CommitMethod(sqlite3_vtab *pVtab){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); UNUSED_PARAMETER(pVtab); - assert( ((Fts3Table *)pVtab)->nPendingData==0 ); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); return SQLITE_OK; } @@ -114009,93 +114700,31 @@ static int fts3CommitMethod(sqlite3_vtab *pVtab){ ** hash-table. Any changes made to the database are reverted by SQLite. */ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ - sqlite3Fts3PendingTermsClear((Fts3Table *)pVtab); + Fts3Table *p = (Fts3Table*)pVtab; + sqlite3Fts3PendingTermsClear(p); + assert( p->inTransaction!=0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); return SQLITE_OK; } /* -** Load the doclist associated with expression pExpr to pExpr->aDoclist. -** The loaded doclist contains positions as well as the document ids. -** This is used by the matchinfo(), snippet() and offsets() auxillary -** functions. +** When called, *ppPoslist must point to the byte immediately following the +** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function +** moves *ppPoslist so that it instead points to the first byte of the +** same position list. */ -SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc; - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); - return rc; -} +static void fts3ReversePoslist(char *pStart, char **ppPoslist){ + char *p = &(*ppPoslist)[-2]; + char c; -SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - char **paDoclist, - int *pnDoclist -){ - int rc; - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); - pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; - rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); - pCsr->eEvalmode = FTS3_EVAL_NEXT; - return rc; -} - -/* -** After ExprLoadDoclist() (see above) has been called, this function is -** used to iterate/search through the position lists that make up the doclist -** stored in pExpr->aDoclist. -*/ -SQLITE_PRIVATE char *sqlite3Fts3FindPositions( - Fts3Expr *pExpr, /* Access this expressions doclist */ - sqlite3_int64 iDocid, /* Docid associated with requested pos-list */ - int iCol /* Column of requested pos-list */ -){ - assert( pExpr->isLoaded ); - if( pExpr->aDoclist ){ - char *pEnd = &pExpr->aDoclist[pExpr->nDoclist]; - char *pCsr; - - if( pExpr->pCurrent==0 ){ - pExpr->pCurrent = pExpr->aDoclist; - pExpr->iCurrent = 0; - pExpr->pCurrent += sqlite3Fts3GetVarint(pExpr->pCurrent,&pExpr->iCurrent); - } - pCsr = pExpr->pCurrent; - assert( pCsr ); - - while( pCsriCurrentiCurrent); - } - pExpr->pCurrent = pCsr; - }else{ - if( pExpr->iCurrent==iDocid ){ - int iThis = 0; - if( iCol<0 ){ - /* If iCol is negative, return a pointer to the start of the - ** position-list (instead of a pointer to the start of a list - ** of offsets associated with a specific column). - */ - return pCsr; - } - while( iThispStart && (c=*p--)==0 ); + while( p>pStart && (*p & 0x80) | c ){ + c = *p--; } - - return 0; + if( p>pStart ){ p = &p[2]; } + while( *p++&0x80 ); + *ppPoslist = p; } /* @@ -114328,8 +114957,34 @@ static int fts3RenameMethod( return rc; } +static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + UNUSED_PARAMETER(iSavepoint); + assert( ((Fts3Table *)pVtab)->inTransaction ); + assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); + TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); + return fts3SyncMethod(pVtab); +} +static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(iSavepoint); + UNUSED_PARAMETER(pVtab); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint-1 ); + return SQLITE_OK; +} +static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(iSavepoint); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint ); + sqlite3Fts3PendingTermsClear(p); + return SQLITE_OK; +} + static const sqlite3_module fts3Module = { - /* iVersion */ 0, + /* iVersion */ 2, /* xCreate */ fts3CreateMethod, /* xConnect */ fts3ConnectMethod, /* xBestIndex */ fts3BestIndexMethod, @@ -114349,6 +115004,9 @@ static const sqlite3_module fts3Module = { /* xRollback */ fts3RollbackMethod, /* xFindFunction */ fts3FindFunctionMethod, /* xRename */ fts3RenameMethod, + /* xSavepoint */ fts3SavepointMethod, + /* xRelease */ fts3ReleaseMethod, + /* xRollbackTo */ fts3RollbackToMethod, }; /* @@ -114395,6 +115053,11 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ sqlite3Fts3IcuTokenizerModule(&pIcu); #endif +#ifdef SQLITE_TEST + rc = sqlite3Fts3InitTerm(db); + if( rc!=SQLITE_OK ) return rc; +#endif + rc = sqlite3Fts3InitAux(db); if( rc!=SQLITE_OK ) return rc; @@ -114470,6 +115133,1308 @@ SQLITE_API int sqlite3_extension_init( } #endif + +/* +** Allocate an Fts3MultiSegReader for each token in the expression headed +** by pExpr. +** +** An Fts3SegReader object is a cursor that can seek or scan a range of +** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple +** Fts3SegReader objects internally to provide an interface to seek or scan +** within the union of all segments of a b-tree. Hence the name. +** +** If the allocated Fts3MultiSegReader just seeks to a single entry in a +** segment b-tree (if the term is not a prefix or it is a prefix for which +** there exists prefix b-tree of the right length) then it may be traversed +** and merged incrementally. Otherwise, it has to be merged into an in-memory +** doclist and then traversed. +*/ +static void fts3EvalAllocateReaders( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pnToken, /* OUT: Total number of tokens in phrase. */ + int *pnOr, /* OUT: Total number of OR nodes in expr. */ + int *pRc +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + *pnToken += nToken; + for(i=0; ipPhrase->aToken[i]; + int rc = sqlite3Fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + } + assert( pExpr->pPhrase->iDoclistToken==0 ); + pExpr->pPhrase->iDoclistToken = -1; + }else{ + *pnOr += (pExpr->eType==FTSQUERY_OR); + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); + } + } +} + +static void fts3EvalPhraseMergeToken( + Fts3Table *pTab, + Fts3Phrase *p, + int iToken, + char *pList, + int nList +){ + assert( iToken!=p->iDoclistToken ); + + if( pList==0 ){ + sqlite3_free(p->doclist.aAll); + p->doclist.aAll = 0; + p->doclist.nAll = 0; + } + + else if( p->iDoclistToken<0 ){ + p->doclist.aAll = pList; + p->doclist.nAll = nList; + } + + else if( p->doclist.aAll==0 ){ + sqlite3_free(pList); + } + + else { + char *pLeft; + char *pRight; + int nLeft; + int nRight; + int nDiff; + + if( p->iDoclistTokendoclist.aAll; + nLeft = p->doclist.nAll; + pRight = pList; + nRight = nList; + nDiff = iToken - p->iDoclistToken; + }else{ + pRight = p->doclist.aAll; + nRight = p->doclist.nAll; + pLeft = pList; + nLeft = nList; + nDiff = p->iDoclistToken - iToken; + } + + fts3DoclistPhraseMerge(pTab->bDescIdx, nDiff, pLeft, nLeft, pRight,&nRight); + sqlite3_free(pLeft); + p->doclist.aAll = pRight; + p->doclist.nAll = nRight; + } + + if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; +} + +static int fts3EvalPhraseLoad( + Fts3Cursor *pCsr, + Fts3Phrase *p +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &p->aToken[iToken]; + assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); + + if( pToken->pSegcsr ){ + int nThis = 0; + char *pThis = 0; + rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis); + if( rc==SQLITE_OK ){ + fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); + } + } + assert( pToken->pSegcsr==0 ); + } + + return rc; +} + +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ + int iToken; + int rc = SQLITE_OK; + + int nMaxUndeferred = pPhrase->iDoclistToken; + char *aPoslist = 0; + int nPoslist = 0; + int iPrev = -1; + + assert( pPhrase->doclist.bFreeList==0 ); + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + Fts3DeferredToken *pDeferred = pToken->pDeferred; + + if( pDeferred ){ + char *pList; + int nList; + rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + if( rc!=SQLITE_OK ) return rc; + + if( pList==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + + }else if( aPoslist==0 ){ + aPoslist = pList; + nPoslist = nList; + + }else{ + char *aOut = pList; + char *p1 = aPoslist; + char *p2 = aOut; + + assert( iPrev>=0 ); + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); + sqlite3_free(aPoslist); + aPoslist = pList; + nPoslist = aOut - aPoslist; + if( nPoslist==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + } + } + iPrev = iToken; + } + } + + if( iPrev>=0 ){ + if( nMaxUndeferred<0 ){ + pPhrase->doclist.pList = aPoslist; + pPhrase->doclist.nList = nPoslist; + pPhrase->doclist.iDocid = pCsr->iPrevId; + pPhrase->doclist.bFreeList = 1; + }else{ + int nDistance; + char *p1; + char *p2; + char *aOut; + + if( nMaxUndeferred>iPrev ){ + p1 = aPoslist; + p2 = pPhrase->doclist.pList; + nDistance = nMaxUndeferred - iPrev; + }else{ + p1 = pPhrase->doclist.pList; + p2 = aPoslist; + nDistance = iPrev - nMaxUndeferred; + } + + aOut = (char *)sqlite3_malloc(nPoslist+8); + if( !aOut ){ + sqlite3_free(aPoslist); + return SQLITE_NOMEM; + } + + pPhrase->doclist.pList = aOut; + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.bFreeList = 1; + pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList); + }else{ + sqlite3_free(aOut); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + } + sqlite3_free(aPoslist); + } + } + + return SQLITE_OK; +} + +/* +** This function is called for each Fts3Phrase in a full-text query +** expression to initialize the mechanism for returning rows. Once this +** function has been called successfully on an Fts3Phrase, it may be +** used with fts3EvalPhraseNext() to iterate through the matching docids. +*/ +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ + int rc; + Fts3PhraseToken *pFirst = &p->aToken[0]; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + + if( pCsr->bDesc==pTab->bDescIdx + && bOptOk==1 + && p->nToken==1 + && pFirst->pSegcsr + && pFirst->pSegcsr->bLookup + ){ + /* Use the incremental approach. */ + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); + rc = sqlite3Fts3MsrIncrStart( + pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n); + p->bIncr = 1; + + }else{ + /* Load the full doclist for the phrase into memory. */ + rc = fts3EvalPhraseLoad(pCsr, p); + p->bIncr = 0; + } + + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); + return rc; +} + +/* +** This function is used to iterate backwards (from the end to start) +** through doclists. +*/ +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* IN/OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + int iMul = 1; + + while( pDociddoclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + + if( p->bIncr ){ + assert( p->nToken==1 ); + assert( pDL->pNextDocid==0 ); + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &pDL->iDocid, &pDL->pList, &pDL->nList + ); + if( rc==SQLITE_OK && !pDL->pList ){ + *pbEof = 1; + } + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; + }else{ + char *pIter; /* Used to iterate through aAll */ + char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + }else{ + pIter = pDL->aAll; + } + + if( pIter>=pEnd ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim2(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim2(). */ + while( pIterpNextDocid = pIter; + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); + *pbEof = 0; + } + } + + return rc; +} + +static void fts3EvalStartReaders( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int bOptOk, + int *pRc +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; + } + pExpr->bDeferred = (i==nToken); + *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase); + }else{ + fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc); + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); + } + } +} + +typedef struct Fts3TokenAndCost Fts3TokenAndCost; +struct Fts3TokenAndCost { + Fts3Phrase *pPhrase; /* The phrase the token belongs to */ + int iToken; /* Position of token in phrase */ + Fts3PhraseToken *pToken; /* The token itself */ + Fts3Expr *pRoot; + int nOvfl; + int iCol; /* The column the token must match */ +}; + +static void fts3EvalTokenCosts( + Fts3Cursor *pCsr, + Fts3Expr *pRoot, + Fts3Expr *pExpr, + Fts3TokenAndCost **ppTC, + Fts3Expr ***ppOr, + int *pRc +){ + if( *pRc==SQLITE_OK && pExpr ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + for(i=0; *pRc==SQLITE_OK && inToken; i++){ + Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pPhrase = pPhrase; + pTC->iToken = i; + pTC->pRoot = pRoot; + pTC->pToken = &pPhrase->aToken[i]; + pTC->iCol = pPhrase->iColumn; + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); + } + }else if( pExpr->eType!=FTSQUERY_NOT ){ + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pLeft; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); + } + } +} + +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + int rc; + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + sqlite3_stmt *pStmt; + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *pEnd; + const char *a; + + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); + if( rc!=SQLITE_OK ) return rc; + a = sqlite3_column_blob(pStmt, 0); + assert( a ); + + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarint(a, &nDoc); + while( anDoc = nDoc; + pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); + assert( pCsr->nRowAvg>0 ); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + + *pnPage = pCsr->nRowAvg; + return SQLITE_OK; +} + +static int fts3EvalSelectDeferred( + Fts3Cursor *pCsr, + Fts3Expr *pRoot, + Fts3TokenAndCost *aTC, + int nTC +){ + int nDocSize = 0; + int nDocEst = 0; + int rc = SQLITE_OK; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int ii; + + int nOvfl = 0; + int nTerm = 0; + + for(ii=0; iinOvfl) + ){ + pTC = &aTC[jj]; + } + } + assert( pTC ); + + /* At this point pTC points to the cheapest remaining token. */ + if( ii==0 ){ + if( pTC->nOvfl ){ + nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; + }else{ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); + + if( rc==SQLITE_OK ){ + nDocEst = fts3DoclistCountDocids(1, pList, nList); + fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); + } + } + }else{ + if( pTC->nOvfl>=(nDocEst*nDocSize) ){ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + } + nDocEst = 1 + (nDocEst/4); + } + pTC->pToken = 0; + } + + return rc; +} + +SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int nToken = 0; + int nOr = 0; + + /* Allocate a MultiSegReader for each token in the expression. */ + fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc); + + /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO: + ** This call will eventually also be responsible for determining which + ** tokens are 'deferred' until the document text is loaded into memory. + ** + ** Each token in each phrase is dealt with using one of the following + ** three strategies: + ** + ** 1. Entire doclist loaded into memory as part of the + ** fts3EvalStartReaders() call. + ** + ** 2. Doclist loaded into memory incrementally, as part of each + ** sqlite3Fts3EvalNext() call. + ** + ** 3. Token doclist is never loaded. Instead, documents are loaded into + ** memory and scanned for the token as part of the sqlite3Fts3EvalNext() + ** call. This is known as a "deferred" token. + */ + + /* If bOptOk is true, check if there are any tokens that should be deferred. + */ + if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){ + Fts3TokenAndCost *aTC; + Fts3Expr **apOr; + aTC = (Fts3TokenAndCost *)sqlite3_malloc( + sizeof(Fts3TokenAndCost) * nToken + + sizeof(Fts3Expr *) * nOr * 2 + ); + apOr = (Fts3Expr **)&aTC[nToken]; + + if( !aTC ){ + rc = SQLITE_NOMEM; + }else{ + int ii; + Fts3TokenAndCost *pTC = aTC; + Fts3Expr **ppOr = apOr; + + fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc); + nToken = pTC-aTC; + nOr = ppOr-apOr; + + if( rc==SQLITE_OK ){ + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iidoclist.bFreeList ){ + sqlite3_free(pPhrase->doclist.pList); + } + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; +} + +static int fts3EvalNearTrim2( + int nNear, + char *aTmp, /* Temporary space to use */ + char **paPoslist, /* IN/OUT: Position list */ + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +){ + int nParam1 = nNear + pPhrase->nToken; + int nParam2 = nNear + *pnToken; + int nNew; + char *p2; + char *pOut; + int res; + + assert( pPhrase->doclist.pList ); + + p2 = pOut = pPhrase->doclist.pList; + res = fts3PoslistNearMerge( + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + ); + if( res ){ + nNew = (pOut - pPhrase->doclist.pList) - 1; + assert( pPhrase->doclist.pList[nNew]=='\0' ); + assert( nNew<=pPhrase->doclist.nList && nNew>0 ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; + } + + return res; +} + +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ + int res = 1; + + /* The following block runs if pExpr is the root of a NEAR query. + ** For example, the query: + ** + ** "w" NEAR "x" NEAR "y" NEAR "z" + ** + ** which is represented in tree form as: + ** + ** | + ** +--NEAR--+ <-- root of NEAR query + ** | | + ** +--NEAR--+ "z" + ** | | + ** +--NEAR--+ "y" + ** | | + ** "w" "x" + ** + ** The right-hand child of a NEAR node is always a phrase. The + ** left-hand child may be either a phrase or a NEAR node. There are + ** no exceptions to this. + */ + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR + && pExpr->bEof==0 + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + int nTmp = 0; /* Bytes of temp space */ + char *aTmp; /* Temp space for PoslistNearMerge() */ + + /* Allocate temporary working space. */ + for(p=pExpr; p->pLeft; p=p->pLeft){ + nTmp += p->pRight->pPhrase->doclist.nList; + } + nTmp += p->pPhrase->doclist.nList; + aTmp = sqlite3_malloc(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; + + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear = p->pParent->nNear; + Fts3Phrase *pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + } + + sqlite3_free(aTmp); + } + + return res; +} + +/* +** This macro is used by the fts3EvalNext() function. The two arguments are +** 64-bit docid values. If the current query is "ORDER BY docid ASC", then +** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then +** it returns (i2 - i1). This allows the same code to be used for merging +** doclists in ascending or descending order. +*/ +#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2)) + +static void fts3EvalNext( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( *pRc==SQLITE_OK ){ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + if( pLeft->bDeferred ){ + fts3EvalNext(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + fts3EvalNext(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + fts3EvalNext(pCsr, pLeft, pRc); + fts3EvalNext(pCsr, pRight, pRc); + + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNext(pCsr, pLeft, pRc); + }else{ + fts3EvalNext(pCsr, pRight, pRc); + } + } + + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNext(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ + fts3EvalNext(pCsr, pRight, pRc); + }else{ + fts3EvalNext(pCsr, pLeft, pRc); + fts3EvalNext(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNext(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNext(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNext(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalZeroPoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } + } + } +} + +static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ + int bHit = 1; + if( *pRc==SQLITE_OK ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + && fts3EvalNearTest(pExpr, pRc) + ); + + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: + ** + ** ... MATCH 'a OR (b NEAR c)' + ** + ** is matched against a row containing: + ** + ** 'a b d e' + ** + ** then any snippet() should ony highlight the "a" term, not the "b" + ** (as "b" is part of a non-matching NEAR clause). + */ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pRight->pPhrase); + } + } + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pPhrase); + } + } + + break; + + case FTSQUERY_OR: { + int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc); + bHit = bHit1 || bHit2; + break; + } + + case FTSQUERY_NOT: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + ); + break; + + default: { + if( pCsr->pDeferred + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) + ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); + if( pExpr->bDeferred ){ + fts3EvalZeroPoslist(pPhrase); + } + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + }else{ + bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); + } + break; + } + } + } + return bHit; +} + +/* +** Return 1 if both of the following are true: +** +** 1. *pRc is SQLITE_OK when this function returns, and +** +** 2. After scanning the current FTS table row for the deferred tokens, +** it is determined that the row does not match the query. +** +** Or, if no error occurs and it seems the current row does match the FTS +** query, return 0. +*/ +static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ + int rc = *pRc; + int bMiss = 0; + if( rc==SQLITE_OK ){ + if( pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + } + bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc)); + sqlite3Fts3FreeDeferredDoclists(pCsr); + *pRc = rc; + } + return (rc==SQLITE_OK && bMiss); +} + +/* +** Advance to the next document that matches the FTS expression in +** Fts3Cursor.pExpr. +*/ +SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Expr *pExpr = pCsr->pExpr; + assert( pCsr->isEof==0 ); + if( pExpr==0 ){ + pCsr->isEof = 1; + }else{ + do { + if( pCsr->isRequireSeek==0 ){ + sqlite3_reset(pCsr->pStmt); + } + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + fts3EvalNext(pCsr, pExpr, &rc); + pCsr->isEof = pExpr->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pExpr->iDocid; + }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) ); + } + return rc; +} + +/* +** Restart interation for expression pExpr so that the next call to +** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental +** loading or merging of phrase doclists for this iteration. +** +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op. If an error occurs within this function, *pRc is set to an +** SQLite error code before returning. +*/ +static void fts3EvalRestart( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + + if( pPhrase ){ + fts3EvalZeroPoslist(pPhrase); + if( pPhrase->bIncr ){ + assert( pPhrase->nToken==1 ); + assert( pPhrase->aToken[0].pSegcsr ); + sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr); + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); + } + + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; + } + + pExpr->iDocid = 0; + pExpr->bEof = 0; + pExpr->bStart = 0; + + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); + fts3EvalRestart(pCsr, pExpr->pRight, pRc); + } +} + +/* +** After allocating the Fts3Expr.aMI[] array for each phrase in the +** expression rooted at pExpr, the cursor iterates through all rows matched +** by pExpr, calling this function for each row. This function increments +** the values in Fts3Expr.aMI[] according to the position-list currently +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** expression nodes. +*/ +static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ + if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase && pPhrase->doclist.pList ){ + int iCol = 0; + char *p = pPhrase->doclist.pList; + + assert( *p ); + while( 1 ){ + u8 c = 0; + int iCnt = 0; + while( 0xFE & (*p | c) ){ + if( (c&0x80)==0 ) iCnt++; + c = *p++ & 0x80; + } + + /* aMI[iCol*3 + 1] = Number of occurrences + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance + */ + pExpr->aMI[iCol*3 + 1] += iCnt; + pExpr->aMI[iCol*3 + 2] += (iCnt>0); + if( *p==0x00 ) break; + p++; + p += sqlite3Fts3GetVarint32(p, &iCol); + } + } + + fts3EvalUpdateCounts(pExpr->pLeft); + fts3EvalUpdateCounts(pExpr->pRight); + } +} + +/* +** Expression pExpr must be of type FTSQUERY_PHRASE. +** +** If it is not already allocated and populated, this function allocates and +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part +** of a NEAR expression, then it also allocates and populates the same array +** for all other phrases that are part of the NEAR expression. +** +** SQLITE_OK is returned if the aMI[] array is successfully allocated and +** populated. Otherwise, if an error occurs, an SQLite error code is returned. +*/ +static int fts3EvalGatherStats( + Fts3Cursor *pCsr, /* Cursor object */ + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pExpr->eType==FTSQUERY_PHRASE ); + if( pExpr->aMI==0 ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + Fts3Expr *pRoot; /* Root of NEAR expression */ + Fts3Expr *p; /* Iterator used for several purposes */ + + sqlite3_int64 iPrevId = pCsr->iPrevId; + sqlite3_int64 iDocid; + u8 bEof; + + /* Find the root of the NEAR expression */ + pRoot = pExpr; + while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + pRoot = pRoot->pParent; + } + iDocid = pRoot->iDocid; + bEof = pRoot->bEof; + assert( pRoot->bStart ); + + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ + for(p=pRoot; p; p=p->pLeft){ + Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); + assert( pE->aMI==0 ); + pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); + if( !pE->aMI ) return SQLITE_NOMEM; + memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + } + + fts3EvalRestart(pCsr, pRoot, &rc); + + while( pCsr->isEof==0 && rc==SQLITE_OK ){ + + do { + /* Ensure the %_content statement is reset. */ + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + + /* Advance to the next document */ + fts3EvalNext(pCsr, pRoot, &rc); + pCsr->isEof = pRoot->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pRoot->iDocid; + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && fts3EvalLoadDeferred(pCsr, &rc) + ); + + if( rc==SQLITE_OK && pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pRoot); + } + } + + pCsr->isEof = 0; + pCsr->iPrevId = iPrevId; + + if( bEof ){ + pRoot->bEof = bEof; + }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidbEof==0 ); + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); + fts3EvalLoadDeferred(pCsr, &rc); + } + } + return rc; +} + +/* +** This function is used by the matchinfo() module to query a phrase +** expression node for the following information: +** +** 1. The total number of occurrences of the phrase in each column of +** the FTS table (considering all rows), and +** +** 2. For each column, the number of rows in the table for which the +** column contains at least one instance of the phrase. +** +** If no error occurs, SQLITE_OK is returned and the values for each column +** written into the array aiOut as follows: +** +** aiOut[iCol*3 + 1] = Number of occurrences +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** +** Caveats: +** +** * If a phrase consists entirely of deferred tokens, then all output +** values are set to the number of documents in the table. In other +** words we assume that very common tokens occur exactly once in each +** column of each row of the table. +** +** * If a phrase contains some deferred tokens (and some non-deferred +** tokens), count the potential occurrence identified by considering +** the non-deferred tokens instead of actual phrase occurrences. +** +** * If the phrase is part of a NEAR expression, then only phrase instances +** that meet the NEAR constraint are included in the counts. +*/ +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Phrase expression */ + u32 *aiOut /* Array to write results into (see above) */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int iCol; + + if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ + assert( pCsr->nDoc>0 ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; + aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; + } + }else{ + rc = fts3EvalGatherStats(pCsr, pExpr); + if( rc==SQLITE_OK ){ + assert( pExpr->aMI ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; + } + } + } + + return rc; +} + +/* +** The expression pExpr passed as the second argument to this function +** must be of type FTSQUERY_PHRASE. +** +** The returned value is either NULL or a pointer to a buffer containing +** a position-list indicating the occurrences of the phrase in column iCol +** of the current row. +** +** More specifically, the returned buffer contains 1 varint for each +** occurence of the phrase in the column, stored using the normal (delta+2) +** compression and is terminated by either an 0x01 or 0x00 byte. For example, +** if the requested column contains "a b X c d X X" and the position-list +** for 'X' is requested, the buffer returned may contain: +** +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** +** This function works regardless of whether or not the phrase is deferred, +** incremental, or neither. +*/ +SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + int iCol /* Column to return position list for */ +){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + char *pIter = pPhrase->doclist.pList; + int iThis; + + assert( iCol>=0 && iColnColumn ); + if( !pIter + || pExpr->bEof + || pExpr->iDocid!=pCsr->iPrevId + || (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) + ){ + return 0; + } + + assert( pPhrase->doclist.nList>0 ); + if( *pIter==0x01 ){ + pIter++; + pIter += sqlite3Fts3GetVarint32(pIter, &iThis); + }else{ + iThis = 0; + } + while( iThisdoclist, and +** * any Fts3MultiSegReader objects held by phrase tokens. +*/ +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ + if( pPhrase ){ + int i; + sqlite3_free(pPhrase->doclist.aAll); + fts3EvalZeroPoslist(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } + } +} + #endif /************** End of fts3.c ************************************************/ @@ -114487,7 +116452,6 @@ SQLITE_API int sqlite3_extension_init( ****************************************************************************** ** */ - #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) @@ -114501,7 +116465,7 @@ struct Fts3auxTable { struct Fts3auxCursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - Fts3SegReaderCursor csr; /* Must be right after "base" */ + Fts3MultiSegReader csr; /* Must be right after "base" */ Fts3SegFilter filter; char *zStop; int nStop; /* Byte-length of string zStop */ @@ -114569,6 +116533,7 @@ static int fts3auxConnectMethod( p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = 1; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); @@ -114815,6 +116780,7 @@ static int fts3auxFilterMethod( int isScan; UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(idxStr); assert( idxStr==0 ); assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0 @@ -114848,7 +116814,7 @@ static int fts3auxFilterMethod( if( pCsr->zStop==0 ) return SQLITE_NOMEM; } - rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL, + rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr ); if( rc==SQLITE_OK ){ @@ -114932,7 +116898,10 @@ SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindFunction */ - 0 /* xRename */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ }; int rc; /* Return code */ @@ -115024,12 +116993,21 @@ SQLITE_API int sqlite3_fts3_enable_parentheses = 0; #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 +/* +** isNot: +** This variable is used by function getNextNode(). When getNextNode() is +** called, it sets ParseContext.isNot to true if the 'next node' is a +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to +** zero. +*/ typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ const char **azCol; /* Array of column names for fts3 table */ int nCol; /* Number of entries in azCol[] */ int iDefaultCol; /* Default column to query */ + int isNot; /* True if getNextNode() sees a unary - */ sqlite3_context *pCtx; /* Write error message here */ int nNest; /* Number of nested brackets */ }; @@ -115115,7 +117093,7 @@ static int getNextToken( iEnd++; } if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ - pRet->pPhrase->isNot = 1; + pParse->isNot = 1; } } nConsumed = iEnd; @@ -115167,36 +117145,55 @@ static int getNextString( char *zTemp = 0; int nTemp = 0; + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + int nToken = 0; + + /* The final Fts3Expr data structure, including the Fts3Phrase, + ** Fts3PhraseToken structures token buffers are all stored as a single + ** allocation so that the expression can be freed with a single call to + ** sqlite3_free(). Setting this up requires a two pass approach. + ** + ** The first pass, in the block below, uses a tokenizer cursor to iterate + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() + ** to assemble data in two dynamic buffers: + ** + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase + ** structure, followed by the array of Fts3PhraseToken + ** structures. This pass only populates the Fts3PhraseToken array. + ** + ** Buffer zTemp: Contains copies of all tokens. + ** + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase + ** structures. + */ rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor); if( rc==SQLITE_OK ){ int ii; pCursor->pTokenizer = pTokenizer; for(ii=0; rc==SQLITE_OK; ii++){ - const char *zToken; - int nToken, iBegin, iEnd, iPos; - rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); + const char *zByte; + int nByte, iBegin, iEnd, iPos; + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); if( rc==SQLITE_OK ){ - int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); - p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken)); - zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken); - if( !p || !zTemp ){ - goto no_mem; - } - if( ii==0 ){ - memset(p, 0, nByte); - p->pPhrase = (Fts3Phrase *)&p[1]; - } - p->pPhrase = (Fts3Phrase *)&p[1]; - memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken)); - p->pPhrase->nToken = ii+1; - p->pPhrase->aToken[ii].n = nToken; - memcpy(&zTemp[nTemp], zToken, nToken); - nTemp += nToken; - if( iEndpPhrase->aToken[ii].isPrefix = 1; - }else{ - p->pPhrase->aToken[ii].isPrefix = 0; - } + Fts3PhraseToken *pToken; + + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); + if( !p ) goto no_mem; + + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); + if( !zTemp ) goto no_mem; + + assert( nToken==ii ); + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; + memset(pToken, 0, sizeof(Fts3PhraseToken)); + + memcpy(&zTemp[nTemp], zByte, nByte); + nTemp += nByte; + + pToken->n = nByte; + pToken->isPrefix = (iEndpPhrase->nToken-1):0) * sizeof(Fts3PhraseToken); - p = fts3ReallocOrFree(p, nByte + nTemp); - if( !p ){ - goto no_mem; - } - if( zTemp ){ - zNew = &(((char *)p)[nByte]); - memcpy(zNew, zTemp, nTemp); - }else{ - memset(p, 0, nByte+nTemp); - } - p->pPhrase = (Fts3Phrase *)&p[1]; - for(jj=0; jjpPhrase->nToken; jj++){ - p->pPhrase->aToken[jj].z = &zNew[nNew]; - nNew += p->pPhrase->aToken[jj].n; - } - sqlite3_free(zTemp); + char *zBuf = 0; + + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); + if( !p ) goto no_mem; + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); p->eType = FTSQUERY_PHRASE; + p->pPhrase = (Fts3Phrase *)&p[1]; p->pPhrase->iColumn = pParse->iDefaultCol; + p->pPhrase->nToken = nToken; + + zBuf = (char *)&p->pPhrase->aToken[nToken]; + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); + + for(jj=0; jjpPhrase->nToken; jj++){ + p->pPhrase->aToken[jj].z = zBuf; + zBuf += p->pPhrase->aToken[jj].n; + } rc = SQLITE_OK; } @@ -115284,6 +117277,8 @@ static int getNextNode( const char *zInput = z; int nInput = n; + pParse->isNot = 0; + /* Skip over any whitespace before checking for a keyword, an open or ** close bracket, or a quoted string. */ @@ -115502,7 +117497,7 @@ static int fts3ExprParse( int isPhrase; if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot + && p->eType==FTSQUERY_PHRASE && pParse->isNot ){ /* Create an implicit NOT operator. */ Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); @@ -115520,7 +117515,6 @@ static int fts3ExprParse( p = pPrev; }else{ int eType = p->eType; - assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot ); isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); /* The isRequirePhrase variable is set to true if a phrase or @@ -115683,9 +117677,11 @@ SQLITE_PRIVATE int sqlite3Fts3ExprParse( */ SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *p){ if( p ){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pRight); - sqlite3_free(p->aDoclist); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); sqlite3_free(p); } } @@ -115742,7 +117738,7 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ Fts3Phrase *pPhrase = pExpr->pPhrase; int i; zBuf = sqlite3_mprintf( - "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot); + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); for(i=0; zBuf && inToken; i++){ zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, pPhrase->aToken[i].n, pPhrase->aToken[i].z, @@ -116289,7 +118285,6 @@ SQLITE_PRIVATE void *sqlite3Fts3HashInsert( - /* ** Class derived from sqlite3_tokenizer */ @@ -116929,12 +118924,12 @@ SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 #endif +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + /* ** Implementation of the SQL scalar function for accessing the underlying @@ -117058,7 +119053,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( ){ int rc; char *z = (char *)zArg; - int n; + int n = 0; char *zCopy; char *zEnd; /* Pointer to nul-term of zCopy */ sqlite3_tokenizer_module *m; @@ -117423,7 +119418,6 @@ SQLITE_PRIVATE int sqlite3Fts3InitHashTable( - typedef struct simple_tokenizer { sqlite3_tokenizer base; char delim[128]; /* flag ASCII delimiters */ @@ -117660,14 +119654,40 @@ SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( */ #define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) +/* +** Under certain circumstances, b-tree nodes (doclists) can be loaded into +** memory incrementally instead of all at once. This can be a big performance +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() +** method before retrieving all query results (as may happen, for example, +** if a query has a LIMIT clause). +** +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful +** for testing purposes. +** +** If this module is built with SQLITE_TEST defined, these constants may +** be overridden at runtime for testing purposes. File fts3_test.c contains +** a Tcl interface to read and write the values. +*/ +#ifdef SQLITE_TEST +int test_fts3_node_chunksize = (4*1024); +int test_fts3_node_chunk_threshold = (4*1024)*4; +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold +#else +# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +#endif + typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; /* -** Data structure used while accumulating terms in the pending-terms hash -** table. The hash table entry maps from term (a string) to a malloc'd -** instance of this structure. +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. */ struct PendingList { int nData; @@ -117698,7 +119718,6 @@ struct Fts3DeferredToken { ** ** sqlite3Fts3SegReaderNew() ** sqlite3Fts3SegReaderFree() -** sqlite3Fts3SegReaderCost() ** sqlite3Fts3SegReaderIterate() ** ** Methods used to manipulate Fts3SegReader structures: @@ -117717,6 +119736,9 @@ struct Fts3SegReader { char *aNode; /* Pointer to node data (or NULL) */ int nNode; /* Size of buffer at aNode (or 0) */ + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + Fts3HashElem **ppNextElem; /* Variables set by fts3SegReaderNext(). These may be read directly @@ -117730,8 +119752,11 @@ struct Fts3SegReader { char *aDoclist; /* Pointer to doclist of current entry */ int nDoclist; /* Size of doclist in current entry */ - /* The following variables are used to iterate through the current doclist */ + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ sqlite3_int64 iDocid; }; @@ -117769,6 +119794,14 @@ struct SegmentWriter { ** fts3NodeAddTerm() ** fts3NodeWrite() ** fts3NodeFree() +** +** When a b+tree is written to the database (either as a result of a merge +** or the pending-terms table being flushed), leaves are written into the +** database file as soon as they are completely populated. The interior of +** the tree is assembled in memory and written out only once all leaves have +** been populated and stored. This is Ok, as the b+-tree fanout is usually +** very large, meaning that the interior of the tree consumes relatively +** little memory. */ struct SegmentNode { SegmentNode *pParent; /* Parent node (or NULL for root node) */ @@ -117799,10 +119832,10 @@ struct SegmentNode { #define SQL_NEXT_SEGMENTS_ID 10 #define SQL_INSERT_SEGDIR 11 #define SQL_SELECT_LEVEL 12 -#define SQL_SELECT_ALL_LEVEL 13 +#define SQL_SELECT_LEVEL_RANGE 13 #define SQL_SELECT_LEVEL_COUNT 14 -#define SQL_SELECT_SEGDIR_COUNT_MAX 15 -#define SQL_DELETE_SEGDIR_BY_LEVEL 16 +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 +#define SQL_DELETE_SEGDIR_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 #define SQL_DELETE_DOCSIZE 19 @@ -117811,6 +119844,11 @@ struct SegmentNode { #define SQL_SELECT_DOCTOTAL 22 #define SQL_REPLACE_DOCTOTAL 23 +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 + +#define SQL_DELETE_SEGDIR_RANGE 26 + /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -117846,10 +119884,11 @@ static int fts3SqlStmt( /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", -/* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", @@ -117859,6 +119898,11 @@ static int fts3SqlStmt( /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", +/* 24 */ "", +/* 25 */ "", + +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", + }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -117915,7 +119959,7 @@ static int fts3SelectDocsize( rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ rc = sqlite3_reset(pStmt); - if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT; + if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT_VTAB; pStmt = 0; }else{ rc = SQLITE_OK; @@ -118014,14 +120058,32 @@ SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *p){ ** 3: end_block ** 4: root */ -SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){ +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( + Fts3Table *p, /* FTS3 table */ + int iIndex, /* Index for p->aIndex[] */ + int iLevel, /* Level to select */ + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +){ int rc; sqlite3_stmt *pStmt = 0; + + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); + assert( iLevel=0 && iIndexnIndex ); + if( iLevel<0 ){ - rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0); + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1); + } }else{ + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); - if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL); + } } *ppStmt = pStmt; return rc; @@ -118136,6 +120198,47 @@ static int fts3PendingListAppend( return 0; } +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} + +/* +** Add an entry to one of the pending-terms hash tables. +*/ +static int fts3PendingTermsAddOne( + Fts3Table *p, + int iCol, + int iPos, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ + const char *zToken, + int nToken +){ + PendingList *pList; + int rc = SQLITE_OK; + + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(pHash, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + return rc; +} + /* ** Tokenize the nul-terminated string zText and add all tokens to the ** pending-terms hash-table. The docid used is that currently stored in @@ -118166,6 +120269,14 @@ static int fts3PendingTermsAdd( assert( pTokenizer && pModule ); + /* If the user has inserted a NULL value, this function may be called with + ** zText==0. In this case, add zero token entries to the hash table and + ** return early. */ + if( zText==0 ){ + *pnWord = 0; + return SQLITE_OK; + } + rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr); if( rc!=SQLITE_OK ){ return rc; @@ -118176,8 +120287,7 @@ static int fts3PendingTermsAdd( while( SQLITE_OK==rc && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) ){ - PendingList *pList; - + int i; if( iPos>=nWord ) nWord = iPos+1; /* Positions cannot be negative; we use -1 as a terminator internally. @@ -118188,22 +120298,19 @@ static int fts3PendingTermsAdd( break; } - pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken); - if( pList ){ - p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); - } - if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ - if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){ - /* Malloc failed while inserting the new entry. This can only - ** happen if there was no previous entry for this token. - */ - assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) ); - sqlite3_free(pList); - rc = SQLITE_NOMEM; - } - } - if( rc==SQLITE_OK ){ - p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + /* Add the term to the terms index */ + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken + ); + + /* Add the term to each of the prefix indexes that it is not too + ** short for. */ + for(i=1; rc==SQLITE_OK && inIndex; i++){ + struct Fts3Index *pIndex = &p->aIndex[i]; + if( nTokennPrefix ) continue; + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + ); } } @@ -118233,14 +120340,19 @@ static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ } /* -** Discard the contents of the pending-terms hash table. +** Discard the contents of the pending-terms hash tables. */ SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ - Fts3HashElem *pElem; - for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ - sqlite3_free(fts3HashData(pElem)); + int i; + for(i=0; inIndex; i++){ + Fts3HashElem *pElem; + Fts3Hash *pHash = &p->aIndex[i].hPending; + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); + } + fts3HashClear(pHash); } - fts3HashClear(&p->pendingTerms); p->nPendingData = 0; } @@ -118256,11 +120368,9 @@ static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){ int i; /* Iterator variable */ for(i=2; inColumn+2; i++){ const char *zText = (const char *)sqlite3_value_text(apVal[i]); - if( zText ){ - int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); - if( rc!=SQLITE_OK ){ - return rc; - } + int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); + if( rc!=SQLITE_OK ){ + return rc; } aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); } @@ -118365,14 +120475,14 @@ static int fts3DeleteAll(Fts3Table *p){ static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ - sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */ + sqlite3_value *pRowid, /* The docid to be deleted */ u32 *aSz /* Sizes of deleted document written here */ ){ int rc; sqlite3_stmt *pSelect; if( *pRC ) return; - rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal); + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; @@ -118398,7 +120508,7 @@ static void fts3DeleteTerms( ** Forward declaration to account for the circular dependency between ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). */ -static int fts3SegmentMerge(Fts3Table *, int); +static int fts3SegmentMerge(Fts3Table *, int, int); /* ** This function allocates a new level iLevel index in the segdir table. @@ -118415,7 +120525,12 @@ static int fts3SegmentMerge(Fts3Table *, int); ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK ** returned. Otherwise, an SQLite error code is returned. */ -static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ +static int fts3AllocateSegdirIdx( + Fts3Table *p, + int iIndex, /* Index for p->aIndex */ + int iLevel, + int *piIdx +){ int rc; /* Return Code */ sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ int iNext = 0; /* Result of query pNextIdx */ @@ -118423,7 +120538,7 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ /* Set variable iNext to the next available segdir index at level iLevel. */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pNextIdx, 1, iLevel); + sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ iNext = sqlite3_column_int(pNextIdx, 0); } @@ -118437,7 +120552,7 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ - rc = fts3SegmentMerge(p, iLevel); + rc = fts3SegmentMerge(p, iIndex, iLevel); *piIdx = 0; }else{ *piIdx = iNext; @@ -118478,7 +120593,8 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( Fts3Table *p, /* FTS3 table handle */ sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ char **paBlob, /* OUT: Blob data in malloc'd buffer */ - int *pnBlob /* OUT: Size of blob data */ + int *pnBlob, /* OUT: Size of blob data */ + int *pnLoad /* OUT: Bytes actually loaded */ ){ int rc; /* Return code */ @@ -118499,11 +120615,16 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( if( rc==SQLITE_OK ){ int nByte = sqlite3_blob_bytes(p->pSegments); + *pnBlob = nByte; if( paBlob ){ char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); if( !aByte ){ rc = SQLITE_NOMEM; }else{ + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ + nByte = FTS3_NODE_CHUNKSIZE; + *pnLoad = nByte; + } rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); memset(&aByte[nByte], 0, FTS3_NODE_PADDING); if( rc!=SQLITE_OK ){ @@ -118513,7 +120634,6 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( } *paBlob = aByte; } - *pnBlob = nByte; } return rc; @@ -118527,13 +120647,55 @@ SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ sqlite3_blob_close(p->pSegments); p->pSegments = 0; } + +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ + int nRead; /* Number of bytes to read */ + int rc; /* Return code */ + + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); + rc = sqlite3_blob_read( + pReader->pBlob, + &pReader->aNode[pReader->nPopulate], + nRead, + pReader->nPopulate + ); + + if( rc==SQLITE_OK ){ + pReader->nPopulate += nRead; + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); + if( pReader->nPopulate==pReader->nNode ){ + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; + pReader->nPopulate = 0; + } + } + return rc; +} + +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ + int rc = SQLITE_OK; + assert( !pReader->pBlob + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) + ); + while( pReader->pBlob && rc==SQLITE_OK + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate + ){ + rc = fts3SegReaderIncrRead(pReader); + } + return rc; +} /* ** Move the iterator passed as the first argument to the next term in the ** segment. If successful, SQLITE_OK is returned. If there is no next term, ** SQLITE_DONE. Otherwise, an SQLite error code. */ -static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ +static int fts3SegReaderNext( + Fts3Table *p, + Fts3SegReader *pReader, + int bIncr +){ + int rc; /* Return code of various sub-routines */ char *pNext; /* Cursor variable */ int nPrefix; /* Number of bytes in term prefix */ int nSuffix; /* Number of bytes in term suffix */ @@ -118545,7 +120707,6 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ } if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ - int rc; /* Return code from Fts3ReadBlock() */ if( fts3SegReaderIsPending(pReader) ){ Fts3HashElem *pElem = *(pReader->ppNextElem); @@ -118565,6 +120726,8 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; } pReader->aNode = 0; @@ -118576,21 +120739,31 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ } rc = sqlite3Fts3ReadBlock( - p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + (bIncr ? &pReader->nPopulate : 0) ); if( rc!=SQLITE_OK ) return rc; + assert( pReader->pBlob==0 ); + if( bIncr && pReader->nPopulatenNode ){ + pReader->pBlob = p->pSegments; + p->pSegments = 0; + } pNext = pReader->aNode; } + + assert( !fts3SegReaderIsPending(pReader) ); + + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); + if( rc!=SQLITE_OK ) return rc; /* Because of the FTS3_NODE_PADDING bytes of padding, the following is - ** safe (no risk of overread) even if the node data is corrupted. - */ + ** safe (no risk of overread) even if the node data is corrupted. */ pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); if( nPrefix<0 || nSuffix<=0 || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] ){ - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } if( nPrefix+nSuffix>pReader->nTermAlloc ){ @@ -118602,6 +120775,10 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ pReader->zTerm = zNew; pReader->nTermAlloc = nNew; } + + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); + if( rc!=SQLITE_OK ) return rc; + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); pReader->nTerm = nPrefix+nSuffix; pNext += nSuffix; @@ -118614,9 +120791,9 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ ** of these statements is untrue, then the data structure is corrupt. */ if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] - || pReader->aDoclist[pReader->nDoclist-1] + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) ){ - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } return SQLITE_OK; } @@ -118625,12 +120802,26 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ ** Set the SegReader to point to the first docid in the doclist associated ** with the current term. */ -static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ - int n; +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; assert( pReader->aDoclist ); assert( !pReader->pOffsetList ); - n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); - pReader->pOffsetList = &pReader->aDoclist[n]; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } + } + return rc; } /* @@ -118643,128 +120834,125 @@ static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ ** *pnOffsetList is set to the length of the set of column-offset ** lists, not including the nul-terminator byte. For example: */ -static void fts3SegReaderNextDocid( - Fts3SegReader *pReader, - char **ppOffsetList, - int *pnOffsetList +static int fts3SegReaderNextDocid( + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ ){ + int rc = SQLITE_OK; char *p = pReader->pOffsetList; char c = 0; - /* Pointer p currently points at the first byte of an offset list. The - ** following two lines advance it to point one byte past the end of - ** the same offset list. - */ - while( *p | c ) c = *p++ & 0x80; - p++; + assert( p ); - /* If required, populate the output variables with a pointer to and the - ** size of the previous offset-list. - */ - if( ppOffsetList ){ - *ppOffsetList = pReader->pOffsetList; - *pnOffsetList = (int)(p - pReader->pOffsetList - 1); - } - - /* If there are no more entries in the doclist, set pOffsetList to - ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and - ** Fts3SegReader.pOffsetList to point to the next offset list before - ** returning. - */ - if( p>=&pReader->aDoclist[pReader->nDoclist] ){ - pReader->pOffsetList = 0; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; + } }else{ - sqlite3_int64 iDelta; - pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); - pReader->iDocid += iDelta; + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; + + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte passed + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + while( p=pEnd ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid -= iDelta; + }else{ + pReader->iDocid += iDelta; + } + } + } } + + return SQLITE_OK; } -/* -** This function is called to estimate the amount of data that will be -** loaded from the disk If SegReaderIterate() is called on this seg-reader, -** in units of average document size. -** -** This can be used as follows: If the caller has a small doclist that -** contains references to N documents, and is considering merging it with -** a large doclist (size X "average documents"), it may opt not to load -** the large doclist if X>N. -*/ -SQLITE_PRIVATE int sqlite3Fts3SegReaderCost( - Fts3Cursor *pCsr, /* FTS3 cursor handle */ - Fts3SegReader *pReader, /* Segment-reader handle */ - int *pnCost /* IN/OUT: Number of bytes read */ + +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( + Fts3Cursor *pCsr, + Fts3MultiSegReader *pMsr, + int *pnOvfl ){ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; - int rc = SQLITE_OK; /* Return code */ - int nCost = 0; /* Cost in bytes to return */ - int pgsz = p->nPgsz; /* Database page size */ + int nOvfl = 0; + int ii; + int rc = SQLITE_OK; + int pgsz = p->nPgsz; - /* If this seg-reader is reading the pending-terms table, or if all data - ** for the segment is stored on the root page of the b-tree, then the cost - ** is zero. In this case all required data is already in main memory. - */ - if( p->bHasStat - && !fts3SegReaderIsPending(pReader) - && !fts3SegReaderIsRootOnly(pReader) - ){ - int nBlob = 0; - sqlite3_int64 iBlock; + assert( p->bHasStat ); + assert( pgsz>0 ); - if( pCsr->nRowAvg==0 ){ - /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ - sqlite3_stmt *pStmt; - sqlite3_int64 nDoc = 0; - sqlite3_int64 nByte = 0; - const char *pEnd; - const char *a; - - rc = sqlite3Fts3SelectDoctotal(p, &pStmt); - if( rc!=SQLITE_OK ) return rc; - a = sqlite3_column_blob(pStmt, 0); - assert( a ); - - pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; - a += sqlite3Fts3GetVarint(a, &nDoc); - while( anRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz); - assert( pCsr->nRowAvg>0 ); - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ) return rc; - } - - /* Assume that a blob flows over onto overflow pages if it is larger - ** than (pgsz-35) bytes in size (the file-format documentation - ** confirms this). - */ - for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ - rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob); - if( rc!=SQLITE_OK ) break; - if( (nBlob+35)>pgsz ){ - int nOvfl = (nBlob + 34)/pgsz; - nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg); + for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ + Fts3SegReader *pReader = pMsr->apSegment[ii]; + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + sqlite3_int64 jj; + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ + int nBlob; + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + nOvfl += (nBlob + 34)/pgsz; + } } } } - - *pnCost += nCost; + *pnOvfl = nOvfl; return rc; } @@ -118777,6 +120965,7 @@ SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ sqlite3_free(pReader->zTerm); if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); + sqlite3_blob_close(pReader->pBlob); } } sqlite3_free(pReader); @@ -118853,24 +121042,42 @@ static int fts3CompareElemByTerm(const void *lhs, const void *rhs){ /* ** This function is used to allocate an Fts3SegReader that iterates through ** a subset of the terms stored in the Fts3Table.pendingTerms array. +** +** If the isPrefixIter parameter is zero, then the returned SegReader iterates +** through each term in the pending-terms table. Or, if isPrefixIter is +** non-zero, it iterates through each term and its prefixes. For example, if +** the pending terms hash table contains the terms "sqlite", "mysql" and +** "firebird", then the iterator visits the following 'terms' (in the order +** shown): +** +** f fi fir fire fireb firebi firebir firebird +** m my mys mysq mysql +** s sq sql sqli sqlit sqlite +** +** Whereas if isPrefixIter is zero, the terms visited are: +** +** firebird mysql sqlite */ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( Fts3Table *p, /* Virtual table handle */ + int iIndex, /* Index for p->aIndex */ const char *zTerm, /* Term to search for */ int nTerm, /* Size of buffer zTerm */ - int isPrefix, /* True for a term-prefix query */ + int bPrefix, /* True for a prefix iterator */ Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ ){ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ int nElem = 0; /* Size of array at aElem */ int rc = SQLITE_OK; /* Return Code */ + Fts3Hash *pHash; - if( isPrefix ){ + pHash = &p->aIndex[iIndex].hPending; + if( bPrefix ){ int nAlloc = 0; /* Size of allocated array at aElem */ Fts3HashElem *pE = 0; /* Iterator variable */ - for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){ + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ char *zKey = (char *)fts3HashKey(pE); int nKey = fts3HashKeysize(pE); if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ @@ -118887,6 +121094,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( } aElem = aElem2; } + aElem[nElem++] = pE; } } @@ -118900,7 +121108,9 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( } }else{ - Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm); + /* The query is a simple term lookup that matches at most one term in + ** the index. All that is required is a straight hash-lookup. */ + Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm); if( pE ){ aElem = &pE; nElem = 1; @@ -118920,7 +121130,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( } } - if( isPrefix ){ + if( bPrefix ){ sqlite3_free(aElem); } *ppReader = pReader; @@ -118984,6 +121194,18 @@ static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ assert( pLhs->aNode && pRhs->aNode ); return rc; } +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} /* ** Compare the term that the Fts3SegReader object passed as the first argument @@ -119512,16 +121734,16 @@ static void fts3SegWriterFree(SegmentWriter *pWriter){ ** The first value in the apVal[] array is assumed to contain an integer. ** This function tests if there exist any documents with docid values that ** are different from that integer. i.e. if deleting the document with docid -** apVal[0] would mean the FTS3 table were empty. +** pRowid would mean the FTS3 table were empty. ** ** If successful, *pisEmpty is set to true if the table is empty except for -** document apVal[0], or false otherwise, and SQLITE_OK is returned. If an +** document pRowid, or false otherwise, and SQLITE_OK is returned. If an ** error occurs, an SQLite error code is returned. */ -static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){ +static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ sqlite3_stmt *pStmt; int rc; - rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, apVal); + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pisEmpty = sqlite3_column_int(pStmt, 0); @@ -119532,21 +121754,30 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){ } /* -** Set *pnSegment to the total number of segments in the database. Set -** *pnMax to the largest segment level in the database (segment levels -** are stored in the 'level' column of the %_segdir table). +** Set *pnMax to the largest segment level in the database for the index +** iIndex. +** +** Segment levels are stored in the 'level' column of the %_segdir table. ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ -static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ +static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ sqlite3_stmt *pStmt; int rc; + assert( iIndex>=0 && iIndexnIndex ); - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0); + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1); if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnSegment = sqlite3_column_int(pStmt, 0); - *pnMax = sqlite3_column_int(pStmt, 1); + *pnMax = sqlite3_column_int(pStmt, 0); } return sqlite3_reset(pStmt); } @@ -119567,6 +121798,7 @@ static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ */ static int fts3DeleteSegdir( Fts3Table *p, /* Virtual table handle */ + int iIndex, /* Index for p->aIndex */ int iLevel, /* Level of %_segdir entries to delete */ Fts3SegReader **apSegment, /* Array of SegReader objects */ int nReader /* Size of array apSegment */ @@ -119589,18 +121821,23 @@ static int fts3DeleteSegdir( return rc; } + assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); if( iLevel==FTS3_SEGCURSOR_ALL ){ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - sqlite3Fts3PendingTermsClear(p); - }else{ - assert( iLevel>=0 ); - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, iLevel); - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1); } + }else{ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); } return rc; @@ -119649,15 +121886,106 @@ static void fts3ColumnFilter( *pnList = nList; } -SQLITE_PRIVATE int sqlite3Fts3SegReaderStart( +/* +** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any +** existing data). Grow the buffer if required. +** +** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered +** trying to resize the buffer, return SQLITE_NOMEM. +*/ +static int fts3MsrBufferData( + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + char *pList, + int nList +){ + if( nList>pMsr->nBuffer ){ + char *pNew; + pMsr->nBuffer = nList*2; + pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + if( !pNew ) return SQLITE_NOMEM; + pMsr->aBuffer = pNew; + } + + memcpy(pMsr->aBuffer, pList, nList); + return SQLITE_OK; +} + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( Fts3Table *p, /* Virtual table handle */ - Fts3SegReaderCursor *pCsr, /* Cursor object */ - Fts3SegFilter *pFilter /* Restrictions on range of iteration */ + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + sqlite3_int64 *piDocid, /* OUT: Docid value */ + char **paPoslist, /* OUT: Pointer to position list */ + int *pnPoslist /* OUT: Size of position list in bytes */ +){ + int nMerge = pMsr->nAdvance; + Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( nMerge==0 ){ + *paPoslist = 0; + return SQLITE_OK; + } + + while( 1 ){ + Fts3SegReader *pSeg; + pSeg = pMsr->apSegment[0]; + + if( pSeg->pOffsetList==0 ){ + *paPoslist = 0; + break; + }else{ + int rc; + char *pList; + int nList; + int j; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( rc==SQLITE_OK + && jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + if( rc!=SQLITE_OK ) return rc; + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); + + if( pMsr->iColFilter>=0 ){ + fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); + } + + if( nList>0 ){ + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + *paPoslist = pMsr->aBuffer; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + }else{ + *paPoslist = pList; + } + *piDocid = iDocid; + *pnPoslist = nList; + break; + } + } + } + + return SQLITE_OK; +} + +static int fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + const char *zTerm, /* Term searched for (or NULL) */ + int nTerm /* Length of zTerm in bytes */ ){ int i; - - /* Initialize the cursor object */ - pCsr->pFilter = pFilter; + int nSeg = pCsr->nSegment; /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of @@ -119665,24 +121993,105 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStart( ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ - for(i=0; inSegment; i++){ - int nTerm = pFilter->nTerm; - const char *zTerm = pFilter->zTerm; + for(i=0; pCsr->bRestart==0 && inSegment; i++){ Fts3SegReader *pSeg = pCsr->apSegment[i]; do { - int rc = fts3SegReaderNext(p, pSeg); + int rc = fts3SegReaderNext(p, pSeg, 0); if( rc!=SQLITE_OK ) return rc; }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); } - fts3SegReaderSort( - pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp); + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); return SQLITE_OK; } +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + Fts3SegFilter *pFilter /* Restrictions on range of iteration */ +){ + pCsr->pFilter = pFilter; + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); +} + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int rc; + int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); + if( rc!=SQLITE_OK ) return rc; + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } + } + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; + + return SQLITE_OK; +} + +/* +** This function is called on a MultiSegReader that has been started using +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also +** have been made. Calling this function puts the MultiSegReader in such +** a state that if the next two calls are: +** +** sqlite3Fts3SegReaderStart() +** sqlite3Fts3SegReaderStep() +** +** then the entire doclist for the term is available in +** MultiSegReader.aDoclist/nDoclist. +*/ +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ + int i; /* Used to iterate through segment-readers */ + + assert( pCsr->zTerm==0 ); + assert( pCsr->nTerm==0 ); + assert( pCsr->aDoclist==0 ); + assert( pCsr->nDoclist==0 ); + + pCsr->nAdvance = 0; + pCsr->bRestart = 1; + for(i=0; inSegment; i++){ + pCsr->apSegment[i]->pOffsetList = 0; + pCsr->apSegment[i]->nOffsetList = 0; + pCsr->apSegment[i]->iDocid = 0; + } + + return SQLITE_OK; +} + + SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( Fts3Table *p, /* Virtual table handle */ - Fts3SegReaderCursor *pCsr /* Cursor object */ + Fts3MultiSegReader *pCsr /* Cursor object */ ){ int rc = SQLITE_OK; @@ -119695,6 +122104,9 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( Fts3SegReader **apSegment = pCsr->apSegment; int nSegment = pCsr->nSegment; Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); if( pCsr->nSegment==0 ) return SQLITE_OK; @@ -119706,7 +122118,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( ** forward. Then sort the list in order of current term again. */ for(i=0; inAdvance; i++){ - rc = fts3SegReaderNext(p, apSegment[i]); + rc = fts3SegReaderNext(p, apSegment[i], 0); if( rc!=SQLITE_OK ) return rc; } fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); @@ -119745,10 +122157,18 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( } assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); - if( nMerge==1 && !isIgnoreEmpty ){ - pCsr->aDoclist = apSegment[0]->aDoclist; + if( nMerge==1 + && !isIgnoreEmpty + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ pCsr->nDoclist = apSegment[0]->nDoclist; - rc = SQLITE_ROW; + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + pCsr->aDoclist = pCsr->aBuffer; + }else{ + pCsr->aDoclist = apSegment[0]->aDoclist; + } + if( rc==SQLITE_OK ) rc = SQLITE_ROW; }else{ int nDoclist = 0; /* Size of doclist */ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ @@ -119758,22 +122178,22 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( ** and a single term returned with the merged doclist. */ for(i=0; ipOffsetList ){ int j; /* Number of segments that share a docid */ char *pList; int nList; int nByte; sqlite3_int64 iDocid = apSegment[0]->iDocid; - fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); j = 1; while( jpOffsetList && apSegment[j]->iDocid==iDocid ){ - fts3SegReaderNextDocid(apSegment[j], 0, 0); + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); j++; } @@ -119782,7 +122202,19 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( } if( !isIgnoreEmpty || nList>0 ){ - nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); + + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + iDelta = iPrev - iDocid; + }else{ + iDelta = iDocid - iPrev; + } + assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); + assert( nDoclist>0 || iDelta==iDocid ); + + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); if( nDoclist+nByte>pCsr->nBuffer ){ char *aNew; pCsr->nBuffer = (nDoclist+nByte)*2; @@ -119792,9 +122224,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( } pCsr->aBuffer = aNew; } - nDoclist += sqlite3Fts3PutVarint( - &pCsr->aBuffer[nDoclist], iDocid-iPrev - ); + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); iPrev = iDocid; if( isRequirePos ){ memcpy(&pCsr->aBuffer[nDoclist], pList, nList); @@ -119803,7 +122233,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( } } - fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp); + fts3SegReaderSort(apSegment, nMerge, j, xCmp); } if( nDoclist>0 ){ pCsr->aDoclist = pCsr->aBuffer; @@ -119817,8 +122247,9 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( return rc; } + SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( - Fts3SegReaderCursor *pCsr /* Cursor object */ + Fts3MultiSegReader *pCsr /* Cursor object */ ){ if( pCsr ){ int i; @@ -119845,43 +122276,56 @@ SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ -static int fts3SegmentMerge(Fts3Table *p, int iLevel){ +static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ - int iNewLevel = 0; /* Level to create new segment at */ + int iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ - Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + int bIgnoreEmpty = 0; /* True to ignore empty segments */ - rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel=0 && iIndexnIndex ); + + rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the the numerically - ** greatest segment level currently present in the database. The index - ** of the new segment is always 0. */ - int nDummy; /* TODO: Remove this */ + ** greatest segment level currently present in the database for this + ** index. The idx of the new segment is always 0. */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel); + rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel); + bIgnoreEmpty = 1; + + }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; + rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx); }else{ - /* This call is to merge all segments at level iLevel. Find the next + /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - iNewLevel = iLevel+1; - rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); + rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx); + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1; } if( rc!=SQLITE_OK ) goto finished; assert( csr.nSegment>0 ); - assert( iNewLevel>=0 ); + assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) ); + assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) ); memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; - filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0); + filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0); rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); while( SQLITE_OK==rc ){ @@ -119893,8 +122337,10 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ if( rc!=SQLITE_OK ) goto finished; assert( pWriter ); - rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment); - if( rc!=SQLITE_OK ) goto finished; + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment); + if( rc!=SQLITE_OK ) goto finished; + } rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); finished: @@ -119905,10 +122351,17 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ /* -** Flush the contents of pendingTerms to a level 0 segment. +** Flush the contents of pendingTerms to level 0 segments. */ SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ - return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); + int rc = SQLITE_OK; + int i; + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + sqlite3Fts3PendingTermsClear(p); + return rc; } /* @@ -120059,6 +122512,23 @@ static void fts3UpdateDocTotals( sqlite3_free(a); } +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ + int i; + int bSeenDone = 0; + int rc = SQLITE_OK; + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } + sqlite3Fts3SegmentsClose(p); + sqlite3Fts3PendingTermsClear(p); + + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; +} + /* ** Handle a 'special' INSERT of the form: ** @@ -120075,12 +122545,7 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; - }else{ - sqlite3Fts3PendingTermsClear(p); - } + rc = fts3DoOptimize(p, 0); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); @@ -120093,44 +122558,9 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = SQLITE_ERROR; } - sqlite3Fts3SegmentsClose(p); return rc; } -/* -** Return the deferred doclist associated with deferred token pDeferred. -** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already -** been called to allocate and populate the doclist. -*/ -SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ - if( pDeferred->pList ){ - *pnByte = pDeferred->pList->nData; - return pDeferred->pList->aData; - } - *pnByte = 0; - return 0; -} - -/* -** Helper fucntion for FreeDeferredDoclists(). This function removes all -** references to deferred doclists from within the tree of Fts3Expr -** structures headed by -*/ -static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ - if( pExpr ){ - fts3DeferredDoclistClear(pExpr->pLeft); - fts3DeferredDoclistClear(pExpr->pRight); - if( pExpr->isLoaded ){ - sqlite3_free(pExpr->aDoclist); - pExpr->isLoaded = 0; - pExpr->aDoclist = 0; - pExpr->nDoclist = 0; - pExpr->pCurrent = 0; - pExpr->iCurrent = 0; - } - } -} - /* ** Delete all cached deferred doclists. Deferred doclists are cached ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. @@ -120138,12 +122568,9 @@ static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ Fts3DeferredToken *pDef; for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); pDef->pList = 0; } - if( pCsr->pDeferred ){ - fts3DeferredDoclistClear(pCsr->pExpr); - } } /* @@ -120155,7 +122582,7 @@ SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ Fts3DeferredToken *pNext; for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ pNext = pDef->pNext; - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); sqlite3_free(pDef); } pCsr->pDeferred = 0; @@ -120220,6 +122647,33 @@ SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ return rc; } +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( + Fts3DeferredToken *p, + char **ppData, + int *pnData +){ + char *pRet; + int nSkip; + sqlite3_int64 dummy; + + *ppData = 0; + *pnData = 0; + + if( p->pList==0 ){ + return SQLITE_OK; + } + + pRet = (char *)sqlite3_malloc(p->pList->nData); + if( !pRet ) return SQLITE_NOMEM; + + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); + *pnData = p->pList->nData - nSkip; + *ppData = pRet; + + memcpy(pRet, &p->pList->aData[nSkip], *pnData); + return SQLITE_OK; +} + /* ** Add an entry for token pToken to the pCsr->pDeferred list. */ @@ -120245,6 +122699,40 @@ SQLITE_PRIVATE int sqlite3Fts3DeferToken( return SQLITE_OK; } +/* +** SQLite value pRowid contains the rowid of a row that may or may not be +** present in the FTS3 table. If it is, delete it and adjust the contents +** of subsiduary data structures accordingly. +*/ +static int fts3DeleteByRowid( + Fts3Table *p, + sqlite3_value *pRowid, + int *pnDoc, + u32 *aSzDel +){ + int isEmpty = 0; + int rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p); + *pnDoc = *pnDoc - 1; + }else{ + sqlite3_int64 iRemove = sqlite3_value_int64(pRowid); + rc = fts3PendingTermsDocid(p, iRemove); + fts3DeleteTerms(&rc, p, pRowid, aSzDel); + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } + } + } + + return rc; +} /* ** This function does the work for the xUpdate method of FTS3 virtual @@ -120260,49 +122748,97 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ - u32 *aSzIns; /* Sizes of inserted documents */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ + int bInsertDone = 0; assert( p->pSegments==0 ); + /* Check for a "special" INSERT operation. One of the form: + ** + ** INSERT INTO xyz(xyz) VALUES('command'); + */ + if( nArg>1 + && sqlite3_value_type(apVal[0])==SQLITE_NULL + && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL + ){ + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; + } + /* Allocate space to hold the change in document sizes */ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); - if( aSzIns==0 ) return SQLITE_NOMEM; + if( aSzIns==0 ){ + rc = SQLITE_NOMEM; + goto update_out; + } aSzDel = &aSzIns[p->nColumn+1]; memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); - /* If this is a DELETE or UPDATE operation, remove the old record. */ - if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ - int isEmpty = 0; - rc = fts3IsEmpty(p, apVal, &isEmpty); - if( rc==SQLITE_OK ){ - if( isEmpty ){ - /* Deleting this row means the whole table is empty. In this case - ** delete the contents of all three tables and throw away any - ** data in the pendingTerms hash table. - */ - rc = fts3DeleteAll(p); + /* If this is an INSERT operation, or an UPDATE that modifies the rowid + ** value, then this operation requires constraint handling. + ** + ** If the on-conflict mode is REPLACE, this means that the existing row + ** should be deleted from the database before inserting the new row. Or, + ** if the on-conflict mode is other than REPLACE, then this method must + ** detect the conflict and return SQLITE_CONSTRAINT before beginning to + ** modify the database file. + */ + if( nArg>1 ){ + /* Find the value object that holds the new rowid value. */ + sqlite3_value *pNewRowid = apVal[3+p->nColumn]; + if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ + pNewRowid = apVal[1]; + } + + if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( + sqlite3_value_type(apVal[0])==SQLITE_NULL + || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) + )){ + /* The new rowid is not NULL (in this case the rowid will be + ** automatically assigned and there is no chance of a conflict), and + ** the statement is either an INSERT or an UPDATE that modifies the + ** rowid column. So if the conflict mode is REPLACE, then delete any + ** existing row with rowid=pNewRowid. + ** + ** Or, if the conflict mode is not REPLACE, insert the new record into + ** the %_content table. If we hit the duplicate rowid constraint (or any + ** other error) while doing so, return immediately. + ** + ** This branch may also run if pNewRowid contains a value that cannot + ** be losslessly converted to an integer. In this case, the eventual + ** call to fts3InsertData() (either just below or further on in this + ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is + ** invoked, it will delete zero rows (since no row will have + ** docid=$pNewRowid if $pNewRowid is not an integer value). + */ + if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ + rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); }else{ - isRemove = 1; - iRemove = sqlite3_value_int64(apVal[0]); - rc = fts3PendingTermsDocid(p, iRemove); - fts3DeleteTerms(&rc, p, apVal, aSzDel); - fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, apVal); - if( p->bHasDocsize ){ - fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, apVal); - } - nChng--; + rc = fts3InsertData(p, apVal, pRowid); + bInsertDone = 1; } } - }else if( sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ - sqlite3_free(aSzIns); - return fts3SpecialInsert(p, apVal[p->nColumn+2]); + } + if( rc!=SQLITE_OK ){ + goto update_out; + } + + /* If this is a DELETE or UPDATE operation, remove the old record. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); + rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); + isRemove = 1; + iRemove = sqlite3_value_int64(apVal[0]); } /* If this is an INSERT or UPDATE operation, insert the new record. */ if( nArg>1 && rc==SQLITE_OK ){ - rc = fts3InsertData(p, apVal, pRowid); + if( bInsertDone==0 ){ + rc = fts3InsertData(p, apVal, pRowid); + if( rc==SQLITE_CONSTRAINT ) rc = SQLITE_CORRUPT_VTAB; + } if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){ rc = fts3PendingTermsDocid(p, *pRowid); } @@ -120319,6 +122855,7 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } + update_out: sqlite3_free(aSzIns); sqlite3Fts3SegmentsClose(p); return rc; @@ -120333,12 +122870,10 @@ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ int rc; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); - if( rc==SQLITE_OK ){ - sqlite3Fts3PendingTermsClear(p); - } + rc = fts3DoOptimize(p, 1); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc2!=SQLITE_OK ) rc = rc2; }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); @@ -120527,51 +123062,6 @@ static int fts3ExprIterate( return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); } -/* -** The argument to this function is always a phrase node. Its doclist -** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes -** to the left of this one in the query tree have already been loaded. -** -** If this phrase node is part of a series of phrase nodes joined by -** NEAR operators (and is not the left-most of said series), then elements are -** removed from the phrases doclist consistent with the NEAR restriction. If -** required, elements may be removed from the doclists of phrases to the -** left of this one that are part of the same series of NEAR operator -** connected phrases. -** -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. -*/ -static int fts3ExprNearTrim(Fts3Expr *pExpr){ - int rc = SQLITE_OK; - Fts3Expr *pParent = pExpr->pParent; - - assert( pExpr->eType==FTSQUERY_PHRASE ); - while( rc==SQLITE_OK - && pParent - && pParent->eType==FTSQUERY_NEAR - && pParent->pRight==pExpr - ){ - /* This expression (pExpr) is the right-hand-side of a NEAR operator. - ** Find the expression to the left of the same operator. - */ - int nNear = pParent->nNear; - Fts3Expr *pLeft = pParent->pLeft; - - if( pLeft->eType!=FTSQUERY_PHRASE ){ - assert( pLeft->eType==FTSQUERY_NEAR ); - assert( pLeft->pRight->eType==FTSQUERY_PHRASE ); - pLeft = pLeft->pRight; - } - - rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear); - - pExpr = pLeft; - pParent = pExpr->pParent; - } - - return rc; -} - /* ** This is an fts3ExprIterate() callback used while loading the doclists ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also @@ -120579,20 +123069,13 @@ static int fts3ExprNearTrim(Fts3Expr *pExpr){ */ static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; UNUSED_PARAMETER(iPhrase); p->nPhrase++; - p->nToken += pExpr->pPhrase->nToken; - - if( pExpr->isLoaded==0 ){ - rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr); - pExpr->isLoaded = 1; - if( rc==SQLITE_OK ){ - rc = fts3ExprNearTrim(pExpr); - } - } + p->nToken += pPhrase->nToken; return rc; } @@ -120766,7 +123249,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ pPhrase->nToken = pExpr->pPhrase->nToken; - pCsr = sqlite3Fts3FindPositions(pExpr, p->pCsr->iPrevId, p->iCol); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); if( pCsr ){ int iFirst = 0; pPhrase->pList = pCsr; @@ -121123,26 +123606,6 @@ static int fts3ColumnlistCount(char **ppCollist){ return nEntry; } -static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){ - char *pCsr = *pp; - while( *pCsr ){ - int nHit; - sqlite3_int64 iCol = 0; - if( *pCsr==0x01 ){ - pCsr++; - pCsr += sqlite3Fts3GetVarint(pCsr, &iCol); - } - nHit = fts3ColumnlistCount(&pCsr); - assert( nHit>0 ); - if( isGlobal ){ - aOut[iCol*3+1]++; - } - aOut[iCol*3] += nHit; - } - pCsr++; - *pp = pCsr; -} - /* ** fts3ExprIterate() callback used to collect the "global" matchinfo stats ** for a single query. @@ -121176,48 +123639,9 @@ static int fts3ExprGlobalHitsCb( void *pCtx /* Pointer to MatchInfo structure */ ){ MatchInfo *p = (MatchInfo *)pCtx; - Fts3Cursor *pCsr = p->pCursor; - char *pIter; - char *pEnd; - char *pFree = 0; - u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; - - assert( pExpr->isLoaded ); - assert( pExpr->eType==FTSQUERY_PHRASE ); - - if( pCsr->pDeferred ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - for(ii=0; iinToken; ii++){ - if( pPhrase->aToken[ii].bFulltext ) break; - } - if( iinToken ){ - int nFree = 0; - int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree); - if( rc!=SQLITE_OK ) return rc; - pIter = pFree; - pEnd = &pFree[nFree]; - }else{ - int iCol; /* Column index */ - for(iCol=0; iColnCol; iCol++){ - aOut[iCol*3 + 1] = (u32)p->nDoc; - aOut[iCol*3 + 2] = (u32)p->nDoc; - } - return SQLITE_OK; - } - }else{ - pIter = pExpr->aDoclist; - pEnd = &pExpr->aDoclist[pExpr->nDoclist]; - } - - /* Fill in the global hit count matrix row for this phrase. */ - while( pIterpCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] + ); } /* @@ -121234,14 +123658,13 @@ static int fts3ExprLocalHitsCb( int iStart = iPhrase * p->nCol * 3; int i; - for(i=0; inCol; i++) p->aMatchinfo[iStart+i*3] = 0; - - if( pExpr->aDoclist ){ + for(i=0; inCol; i++){ char *pCsr; - - pCsr = sqlite3Fts3FindPositions(pExpr, p->pCursor->iPrevId, -1); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i); if( pCsr ){ - fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0); + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); + }else{ + p->aMatchinfo[iStart+i*3] = 0; } } @@ -121311,7 +123734,7 @@ static int fts3MatchinfoSelectDoctotal( a = sqlite3_column_blob(pStmt, 0); a += sqlite3Fts3GetVarint(a, &nDoc); - if( nDoc==0 ) return SQLITE_CORRUPT; + if( nDoc==0 ) return SQLITE_CORRUPT_VTAB; *pnDoc = (u32)nDoc; if( paLen ) *paLen = a; @@ -121327,9 +123750,8 @@ static int fts3MatchinfoSelectDoctotal( typedef struct LcsIterator LcsIterator; struct LcsIterator { Fts3Expr *pExpr; /* Pointer to phrase expression */ - char *pRead; /* Cursor used to iterate through aDoclist */ int iPosOffset; /* Tokens count up to end of this phrase */ - int iCol; /* Current column number */ + char *pRead; /* Cursor used to iterate through aDoclist */ int iPos; /* Current position */ }; @@ -121360,17 +123782,10 @@ static int fts3LcsIteratorAdvance(LcsIterator *pIter){ int rc = 0; pRead += sqlite3Fts3GetVarint(pRead, &iRead); - if( iRead==0 ){ - pIter->iCol = LCS_ITERATOR_FINISHED; + if( iRead==0 || iRead==1 ){ + pRead = 0; rc = 1; }else{ - if( iRead==1 ){ - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - pIter->iCol = (int)iRead; - pIter->iPos = pIter->iPosOffset; - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - rc = 1; - } pIter->iPos += (int)(iRead-2); } @@ -121402,42 +123817,34 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ if( !aIter ) return SQLITE_NOMEM; memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; nToken -= pIter->pExpr->pPhrase->nToken; pIter->iPosOffset = nToken; - pIter->pRead = sqlite3Fts3FindPositions(pIter->pExpr, pCsr->iPrevId, -1); - if( pIter->pRead ){ - pIter->iPos = pIter->iPosOffset; - fts3LcsIteratorAdvance(&aIter[i]); - }else{ - pIter->iCol = LCS_ITERATOR_FINISHED; - } } for(iCol=0; iColnCol; iCol++){ int nLcs = 0; /* LCS value for this column */ int nLive = 0; /* Number of iterators in aIter not at EOF */ - /* Loop through the iterators in aIter[]. Set nLive to the number of - ** iterators that point to a position-list corresponding to column iCol. - */ for(i=0; inPhrase; i++){ - assert( aIter[i].iCol>=iCol ); - if( aIter[i].iCol==iCol ) nLive++; + LcsIterator *pIt = &aIter[i]; + pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol); + if( pIt->pRead ){ + pIt->iPos = pIt->iPosOffset; + fts3LcsIteratorAdvance(&aIter[i]); + nLive++; + } } - /* The following loop runs until all iterators in aIter[] have finished - ** iterating through positions in column iCol. Exactly one of the - ** iterators is advanced each time the body of the loop is run. - */ while( nLive>0 ){ LcsIterator *pAdv = 0; /* The iterator to advance by one position */ int nThisLcs = 0; /* LCS for the current iterator positions */ for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; - if( iCol!=pIter->iCol ){ + if( pIter->pRead==0 ){ /* This iterator is already at EOF for this column. */ nThisLcs = 0; }else{ @@ -121503,7 +123910,7 @@ static int fts3MatchinfoValues( case FTS3_MATCHINFO_NDOC: if( bGlobal ){ - sqlite3_int64 nDoc; + sqlite3_int64 nDoc = 0; rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0); pInfo->aMatchinfo[0] = (u32)nDoc; } @@ -121759,6 +124166,7 @@ struct TermOffset { }; struct TermOffsetCtx { + Fts3Cursor *pCsr; int iCol; /* Column of table to populate aTerm for */ int iTerm; sqlite3_int64 iDocid; @@ -121776,7 +124184,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ int iPos = 0; /* First position in position-list */ UNUSED_PARAMETER(iPhrase); - pList = sqlite3Fts3FindPositions(pExpr, p->iDocid, p->iCol); + pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); nTerm = pExpr->pPhrase->nToken; if( pList ){ fts3GetDeltaPosition(&pList, &iPos); @@ -121829,6 +124237,7 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( goto offsets_out; } sCtx.iDocid = pCsr->iPrevId; + sCtx.pCsr = pCsr; /* Loop through the table columns, appending offset information to ** string-buffer res for each column. @@ -121904,7 +124313,7 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( ); rc = fts3StringAppend(&res, aBuffer, -1); }else if( rc==SQLITE_DONE ){ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } } } @@ -122492,17 +124901,17 @@ nodeAcquire( if( pNode && iNode==1 ){ pRtree->iDepth = readInt16(pNode->zData); if( pRtree->iDepth>RTREE_MAX_DEPTH ){ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } } /* If no error has occurred so far, check if the "number of entries" ** field on the node is too large. If so, set the return code to - ** SQLITE_CORRUPT. + ** SQLITE_CORRUPT_VTAB. */ if( pNode && rc==SQLITE_OK ){ if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } } @@ -122510,7 +124919,7 @@ nodeAcquire( if( pNode!=0 ){ nodeHashInsert(pRtree, pNode); }else{ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } *ppNode = pNode; }else{ @@ -123037,7 +125446,7 @@ static int nodeRowidIndex( return SQLITE_OK; } } - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } /* @@ -123396,7 +125805,7 @@ static float cellArea(Rtree *pRtree, RtreeCell *p){ float area = 1.0; int ii; for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - area = area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); + area = (float)(area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); } return area; } @@ -123409,7 +125818,7 @@ static float cellMargin(Rtree *pRtree, RtreeCell *p){ float margin = 0.0; int ii; for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); + margin += (float)(DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); } return margin; } @@ -123494,7 +125903,7 @@ static float cellOverlap( o = 0.0; break; }else{ - o = o * (x2-x1); + o = o * (float)(x2-x1); } } overlap += o; @@ -123513,12 +125922,12 @@ static float cellOverlapEnlargement( int nCell, int iExclude ){ - float before; - float after; + double before; + double after; before = cellOverlap(pRtree, p, aCell, nCell, iExclude); cellUnion(pRtree, p, pInsert); after = cellOverlap(pRtree, p, aCell, nCell, iExclude); - return after-before; + return (float)(after-before); } #endif @@ -123540,11 +125949,11 @@ static int ChooseLeaf( for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){ int iCell; - sqlite3_int64 iBest; + sqlite3_int64 iBest = 0; - float fMinGrowth; - float fMinArea; - float fMinOverlap; + float fMinGrowth = 0.0; + float fMinArea = 0.0; + float fMinOverlap = 0.0; int nCell = NCELL(pNode); RtreeCell cell; @@ -123632,7 +126041,7 @@ static int AdjustTree( int iCell; if( nodeParentIndex(pRtree, p, &iCell) ){ - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } nodeGetCell(pRtree, pParent, iCell, &cell); @@ -123974,9 +126383,9 @@ static int splitNodeStartree( int *aSpare; int ii; - int iBestDim; - int iBestSplit; - float fBestMargin; + int iBestDim = 0; + int iBestSplit = 0; + float fBestMargin = 0.0; int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); @@ -123998,9 +126407,9 @@ static int splitNodeStartree( for(ii=0; iinDim; ii++){ float margin = 0.0; - float fBestOverlap; - float fBestArea; - int iBestLeft; + float fBestOverlap = 0.0; + float fBestArea = 0.0; + int iBestLeft = 0; int nLeft; for( @@ -124304,7 +126713,7 @@ static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ } rc = sqlite3_reset(pRtree->pReadParent); if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT; + if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT_VTAB; pChild = pChild->pParent; } return rc; @@ -124315,7 +126724,7 @@ static int deleteCell(Rtree *, RtreeNode *, int, int); static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ int rc; int rc2; - RtreeNode *pParent; + RtreeNode *pParent = 0; int iCell; assert( pNode->nRef==1 ); @@ -124463,19 +126872,19 @@ static int Reinsert( } aOrder[ii] = ii; for(iDim=0; iDimnDim; iDim++){ - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); + aCenterCoord[iDim] += (float)DCOORD(aCell[ii].aCoord[iDim*2]); + aCenterCoord[iDim] += (float)DCOORD(aCell[ii].aCoord[iDim*2+1]); } } for(iDim=0; iDimnDim; iDim++){ - aCenterCoord[iDim] = aCenterCoord[iDim]/((float)nCell*2.0); + aCenterCoord[iDim] = (float)(aCenterCoord[iDim]/((float)nCell*2.0)); } for(ii=0; iinDim; iDim++){ - float coord = DCOORD(aCell[ii].aCoord[iDim*2+1]) - - DCOORD(aCell[ii].aCoord[iDim*2]); + float coord = (float)(DCOORD(aCell[ii].aCoord[iDim*2+1]) - + DCOORD(aCell[ii].aCoord[iDim*2])); aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); } } @@ -124574,10 +126983,10 @@ static int reinsertNodeContent(Rtree *pRtree, RtreeNode *pNode){ /* Find a node to store this cell in. pNode->iNode currently contains ** the height of the sub-tree headed by the cell. */ - rc = ChooseLeaf(pRtree, &cell, pNode->iNode, &pInsert); + rc = ChooseLeaf(pRtree, &cell, (int)pNode->iNode, &pInsert); if( rc==SQLITE_OK ){ int rc2; - rc = rtreeInsertCell(pRtree, pInsert, &cell, pNode->iNode); + rc = rtreeInsertCell(pRtree, pInsert, &cell, (int)pNode->iNode); rc2 = nodeRelease(pRtree, pInsert); if( rc==SQLITE_OK ){ rc = rc2; @@ -124600,6 +127009,90 @@ static int newRowid(Rtree *pRtree, i64 *piRowid){ return rc; } +/* +** Remove the entry with rowid=iDelete from the r-tree structure. +*/ +static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ + int rc; /* Return code */ + RtreeNode *pLeaf; /* Leaf node containing record iDelete */ + int iCell; /* Index of iDelete cell in pLeaf */ + RtreeNode *pRoot; /* Root node of rtree structure */ + + + /* Obtain a reference to the root node to initialise Rtree.iDepth */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + + /* Obtain a reference to the leaf node that contains the entry + ** about to be deleted. + */ + if( rc==SQLITE_OK ){ + rc = findLeafNode(pRtree, iDelete, &pLeaf); + } + + /* Delete the cell in question from the leaf node. */ + if( rc==SQLITE_OK ){ + int rc2; + rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); + if( rc==SQLITE_OK ){ + rc = deleteCell(pRtree, pLeaf, iCell, 0); + } + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + + /* Delete the corresponding entry in the _rowid table. */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); + sqlite3_step(pRtree->pDeleteRowid); + rc = sqlite3_reset(pRtree->pDeleteRowid); + } + + /* Check if the root node now has exactly one child. If so, remove + ** it, schedule the contents of the child for reinsertion and + ** reduce the tree height by one. + ** + ** This is equivalent to copying the contents of the child into + ** the root node (the operation that Gutman's paper says to perform + ** in this scenario). + */ + if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ + int rc2; + RtreeNode *pChild; + i64 iChild = nodeGetRowid(pRtree, pRoot, 0); + rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); + if( rc==SQLITE_OK ){ + rc = removeNode(pRtree, pChild, pRtree->iDepth-1); + } + rc2 = nodeRelease(pRtree, pChild); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ){ + pRtree->iDepth--; + writeInt16(pRoot->zData, pRtree->iDepth); + pRoot->isDirty = 1; + } + } + + /* Re-insert the contents of any underfull nodes removed from the tree. */ + for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ + if( rc==SQLITE_OK ){ + rc = reinsertNodeContent(pRtree, pLeaf); + } + pRtree->pDeleted = pLeaf->pNext; + sqlite3_free(pLeaf); + } + + /* Release the reference to the root node. */ + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRoot); + }else{ + nodeRelease(pRtree, pRoot); + } + + return rc; +} + /* ** The xUpdate method for rtree module virtual tables. */ @@ -124611,103 +127104,25 @@ static int rtreeUpdate( ){ Rtree *pRtree = (Rtree *)pVtab; int rc = SQLITE_OK; + RtreeCell cell; /* New cell to insert if nData>1 */ + int bHaveRowid = 0; /* Set to 1 after new rowid is determined */ rtreeReference(pRtree); - assert(nData>=1); - /* If azData[0] is not an SQL NULL value, it is the rowid of a - ** record to delete from the r-tree table. The following block does - ** just that. + /* Constraint handling. A write operation on an r-tree table may return + ** SQLITE_CONSTRAINT for two reasons: + ** + ** 1. A duplicate rowid value, or + ** 2. The supplied data violates the "x2>=x1" constraint. + ** + ** In the first case, if the conflict-handling mode is REPLACE, then + ** the conflicting row can be removed before proceeding. In the second + ** case, SQLITE_CONSTRAINT must be returned regardless of the + ** conflict-handling mode specified by the user. */ - if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ - i64 iDelete; /* The rowid to delete */ - RtreeNode *pLeaf; /* Leaf node containing record iDelete */ - int iCell; /* Index of iDelete cell in pLeaf */ - RtreeNode *pRoot; - - /* Obtain a reference to the root node to initialise Rtree.iDepth */ - rc = nodeAcquire(pRtree, 1, 0, &pRoot); - - /* Obtain a reference to the leaf node that contains the entry - ** about to be deleted. - */ - if( rc==SQLITE_OK ){ - iDelete = sqlite3_value_int64(azData[0]); - rc = findLeafNode(pRtree, iDelete, &pLeaf); - } - - /* Delete the cell in question from the leaf node. */ - if( rc==SQLITE_OK ){ - int rc2; - rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); - if( rc==SQLITE_OK ){ - rc = deleteCell(pRtree, pLeaf, iCell, 0); - } - rc2 = nodeRelease(pRtree, pLeaf); - if( rc==SQLITE_OK ){ - rc = rc2; - } - } - - /* Delete the corresponding entry in the _rowid table. */ - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); - sqlite3_step(pRtree->pDeleteRowid); - rc = sqlite3_reset(pRtree->pDeleteRowid); - } - - /* Check if the root node now has exactly one child. If so, remove - ** it, schedule the contents of the child for reinsertion and - ** reduce the tree height by one. - ** - ** This is equivalent to copying the contents of the child into - ** the root node (the operation that Gutman's paper says to perform - ** in this scenario). - */ - if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ - int rc2; - RtreeNode *pChild; - i64 iChild = nodeGetRowid(pRtree, pRoot, 0); - rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); - if( rc==SQLITE_OK ){ - rc = removeNode(pRtree, pChild, pRtree->iDepth-1); - } - rc2 = nodeRelease(pRtree, pChild); - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK ){ - pRtree->iDepth--; - writeInt16(pRoot->zData, pRtree->iDepth); - pRoot->isDirty = 1; - } - } - - /* Re-insert the contents of any underfull nodes removed from the tree. */ - for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ - if( rc==SQLITE_OK ){ - rc = reinsertNodeContent(pRtree, pLeaf); - } - pRtree->pDeleted = pLeaf->pNext; - sqlite3_free(pLeaf); - } - - /* Release the reference to the root node. */ - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pRoot); - }else{ - nodeRelease(pRtree, pRoot); - } - } - - /* If the azData[] array contains more than one element, elements - ** (azData[2]..azData[argc-1]) contain a new record to insert into - ** the r-tree structure. - */ - if( rc==SQLITE_OK && nData>1 ){ - /* Insert a new record into the r-tree */ - RtreeCell cell; + if( nData>1 ){ int ii; - RtreeNode *pLeaf; /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. */ assert( nData==(pRtree->nDim*2 + 3) ); @@ -124731,18 +127146,49 @@ static int rtreeUpdate( } } - /* Figure out the rowid of the new row. */ - if( sqlite3_value_type(azData[2])==SQLITE_NULL ){ - rc = newRowid(pRtree, &cell.iRowid); - }else{ + /* If a rowid value was supplied, check if it is already present in + ** the table. If so, the constraint has failed. */ + if( sqlite3_value_type(azData[2])!=SQLITE_NULL ){ cell.iRowid = sqlite3_value_int64(azData[2]); - sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); - if( SQLITE_ROW==sqlite3_step(pRtree->pReadRowid) ){ - sqlite3_reset(pRtree->pReadRowid); - rc = SQLITE_CONSTRAINT; - goto constraint; + if( sqlite3_value_type(azData[0])==SQLITE_NULL + || sqlite3_value_int64(azData[0])!=cell.iRowid + ){ + int steprc; + sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = SQLITE_CONSTRAINT; + goto constraint; + } + } } - rc = sqlite3_reset(pRtree->pReadRowid); + bHaveRowid = 1; + } + } + + /* If azData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ + rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(azData[0])); + } + + /* If the azData[] array contains more than one element, elements + ** (azData[2]..azData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf; + + /* Figure out the rowid of the new row. */ + if( bHaveRowid==0 ){ + rc = newRowid(pRtree, &cell.iRowid); } *pRowid = cell.iRowid; @@ -124787,7 +127233,7 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ } static sqlite3_module rtreeModule = { - 0, /* iVersion */ + 0, /* iVersion */ rtreeCreate, /* xCreate - create a table */ rtreeConnect, /* xConnect - connect to an existing table */ rtreeBestIndex, /* xBestIndex - Determine search strategy */ @@ -124806,7 +127252,10 @@ static sqlite3_module rtreeModule = { 0, /* xCommit - commit transaction */ 0, /* xRollback - rollback transaction */ 0, /* xFindFunction - function overloading */ - rtreeRename /* xRename - rename the table */ + rtreeRename, /* xRename - rename the table */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ }; static int rtreeSqlInit( @@ -124926,7 +127375,7 @@ static int getNodeSize( int rc; char *zSql; if( isCreate ){ - int iPageSize; + int iPageSize = 0; zSql = sqlite3_mprintf("PRAGMA %Q.page_size", pRtree->zDb); rc = getIntFromStmt(db, zSql, &iPageSize); if( rc==SQLITE_OK ){ @@ -124983,6 +127432,8 @@ static int rtreeInit( return SQLITE_ERROR; } + sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + /* Allocate the sqlite3_vtab structure */ nDb = strlen(argv[1]); nName = strlen(argv[2]); @@ -125727,10 +128178,7 @@ SQLITE_API int sqlite3_extension_init( ** ************************************************************************* ** This file implements a tokenizer for fts3 based on the ICU library. -** -** $Id: fts3_icu.c,v 1.3 2008/09/01 18:34:20 danielk1977 Exp $ */ - #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #ifdef SQLITE_ENABLE_ICU diff --git a/harbour/contrib/3rd/sqlite3/sqlite3.dif b/harbour/contrib/3rd/sqlite3/sqlite3.dif index 7ca849454a..8aaadd4fd0 100644 --- a/harbour/contrib/3rd/sqlite3/sqlite3.dif +++ b/harbour/contrib/3rd/sqlite3/sqlite3.dif @@ -1,7 +1,7 @@ diff -urN sqlite3.orig/sqlite3.c sqlite3/sqlite3.c ---- sqlite3.orig/sqlite3.c 2011-05-26 23:29:45.837407067 +0200 -+++ sqlite3/sqlite3.c 2011-05-26 23:29:45.977411381 +0200 -@@ -24781,7 +24781,9 @@ +--- sqlite3.orig/sqlite3.c 2011-07-27 18:12:09.104445521 +0200 ++++ sqlite3/sqlite3.c 2011-07-27 18:12:09.244446575 +0200 +@@ -25143,7 +25143,9 @@ case ENODEV: case ENXIO: case ENOENT: @@ -11,16 +11,7 @@ diff -urN sqlite3.orig/sqlite3.c sqlite3/sqlite3.c case ENOSYS: /* these should force the client to close the file and reconnect */ -@@ -26047,7 +26049,7 @@ - */ - if( pFile->eFileLock > NO_LOCK ){ - pFile->eFileLock = eFileLock; --#if !OS_VXWORKS -+#if !OS_VXWORKS && !__minix - /* Always update the timestamp on the old file */ - utimes(zLockFile, NULL); - #endif -@@ -27520,7 +27522,11 @@ +@@ -27884,7 +27886,11 @@ ** is the same technique used by glibc to implement posix_fallocate() ** on systems that do not have a real fallocate() system call. */ diff --git a/harbour/contrib/3rd/sqlite3/sqlite3.h b/harbour/contrib/3rd/sqlite3/sqlite3.h index 85e916c5e5..ed9edbd202 100644 --- a/harbour/contrib/3rd/sqlite3/sqlite3.h +++ b/harbour/contrib/3rd/sqlite3/sqlite3.h @@ -107,9 +107,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.6.3" -#define SQLITE_VERSION_NUMBER 3007006 -#define SQLITE_SOURCE_ID "2011-05-19 13:26:54 ed1da510a239ea767a01dc332b667119fa3c908e" +#define SQLITE_VERSION "3.7.7.1" +#define SQLITE_VERSION_NUMBER 3007007 +#define SQLITE_SOURCE_ID "2011-06-28 17:39:05 af0d91adf497f5f36ec3813f04235a6e195a605f" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -310,7 +310,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** argument. ^If the callback function of the 3rd argument to ** sqlite3_exec() is not NULL, then it is invoked for each result row ** coming out of the evaluated SQL statements. ^The 4th argument to -** to sqlite3_exec() is relayed through to the 1st argument of each +** sqlite3_exec() is relayed through to the 1st argument of each ** callback invocation. ^If the callback pointer to sqlite3_exec() ** is NULL, then no callback is ever invoked and result rows are ** ignored. @@ -375,7 +375,8 @@ SQLITE_API int sqlite3_exec( ** ** New error codes may be added in future versions of SQLite. ** -** See also: [SQLITE_IOERR_READ | extended result codes] +** See also: [SQLITE_IOERR_READ | extended result codes], +** [sqlite3_vtab_on_conflict()] [SQLITE_ROLLBACK | result codes]. */ #define SQLITE_OK 0 /* Successful result */ /* beginning-of-error-codes */ @@ -452,17 +453,21 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8)) #define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8)) #define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) +#define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) +#define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) +#define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) +#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations ** ** These bit values are intended for use in the ** 3rd parameter to the [sqlite3_open_v2()] interface and -** in the 4th parameter to the xOpen method of the -** [sqlite3_vfs] object. +** in the 4th parameter to the [sqlite3_vfs.xOpen] method. */ #define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_READWRITE 0x00000002 /* Ok for sqlite3_open_v2() */ @@ -470,6 +475,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_OPEN_DELETEONCLOSE 0x00000008 /* VFS only */ #define SQLITE_OPEN_EXCLUSIVE 0x00000010 /* VFS only */ #define SQLITE_OPEN_AUTOPROXY 0x00000020 /* VFS only */ +#define SQLITE_OPEN_URI 0x00000040 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_MAIN_DB 0x00000100 /* VFS only */ #define SQLITE_OPEN_TEMP_DB 0x00000200 /* VFS only */ #define SQLITE_OPEN_TRANSIENT_DB 0x00000400 /* VFS only */ @@ -580,17 +586,18 @@ struct sqlite3_file { /* ** CAPI3REF: OS Interface File Virtual Methods Object ** -** Every file opened by the [sqlite3_vfs] xOpen method populates an +** Every file opened by the [sqlite3_vfs.xOpen] method populates an ** [sqlite3_file] object (or, more commonly, a subclass of the ** [sqlite3_file] object) with a pointer to an instance of this object. ** This object defines the methods used to perform various operations ** against the open file represented by the [sqlite3_file] object. ** -** If the xOpen method sets the sqlite3_file.pMethods element +** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element ** to a non-NULL pointer, then the sqlite3_io_methods.xClose method -** may be invoked even if the xOpen reported that it failed. The -** only way to prevent a call to xClose following a failed xOpen -** is for the xOpen to set the sqlite3_file.pMethods element to NULL. +** may be invoked even if the [sqlite3_vfs.xOpen] reported that it failed. The +** only way to prevent a call to xClose following a failed [sqlite3_vfs.xOpen] +** is for the [sqlite3_vfs.xOpen] to set the sqlite3_file.pMethods element +** to NULL. ** ** The flags argument to xSync may be one of [SQLITE_SYNC_NORMAL] or ** [SQLITE_SYNC_FULL]. The first choice is the normal fsync(). @@ -762,7 +769,8 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** ** An instance of the sqlite3_vfs object defines the interface between ** the SQLite core and the underlying operating system. The "vfs" -** in the name of the object stands for "virtual file system". +** in the name of the object stands for "virtual file system". See +** the [VFS | VFS documentation] for further information. ** ** The value of the iVersion field is initially 1 but may be larger in ** future versions of SQLite. Additional fields may be appended to this @@ -791,6 +799,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** The zName field holds the name of the VFS module. The name must ** be unique across all VFS modules. ** +** [[sqlite3_vfs.xOpen]] ** ^SQLite guarantees that the zFilename parameter to xOpen ** is either a NULL pointer or string obtained ** from xFullPathname() with an optional suffix added. @@ -868,6 +877,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** element will be valid after xOpen returns regardless of the success ** or failure of the xOpen call. ** +** [[sqlite3_vfs.xAccess]] ** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] @@ -892,7 +902,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** method returns a Julian Day Number for the current date and time as ** a floating point value. ** ^The xCurrentTimeInt64() method returns, as an integer, the Julian -** Day Number multipled by 86400000 (the number of milliseconds in +** Day Number multiplied by 86400000 (the number of milliseconds in ** a 24-hour day). ** ^SQLite will use the xCurrentTimeInt64() method to get the current ** date and time if that method is available (if iVersion is 2 or @@ -1114,9 +1124,9 @@ SQLITE_API int sqlite3_os_end(void); ** implementation of an application-defined [sqlite3_os_init()]. ** ** The first argument to sqlite3_config() is an integer -** [SQLITE_CONFIG_SINGLETHREAD | configuration option] that determines +** [configuration option] that determines ** what property of SQLite is to be configured. Subsequent arguments -** vary depending on the [SQLITE_CONFIG_SINGLETHREAD | configuration option] +** vary depending on the [configuration option] ** in the first argument. ** ** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. @@ -1226,6 +1236,7 @@ struct sqlite3_mem_methods { /* ** CAPI3REF: Configuration Options +** KEYWORDS: {configuration option} ** ** These constants are the available integer configuration options that ** can be passed as the first argument to the [sqlite3_config()] interface. @@ -1238,7 +1249,7 @@ struct sqlite3_mem_methods { ** is invoked. ** **
-**
SQLITE_CONFIG_SINGLETHREAD
+** [[SQLITE_CONFIG_SINGLETHREAD]]
SQLITE_CONFIG_SINGLETHREAD
**
There are no arguments to this option. ^This option sets the ** [threading mode] to Single-thread. In other words, it disables ** all mutexing and puts SQLite into a mode where it can only be used @@ -1249,7 +1260,7 @@ struct sqlite3_mem_methods { ** [SQLITE_ERROR] if called with the SQLITE_CONFIG_SINGLETHREAD ** configuration option.
** -**
SQLITE_CONFIG_MULTITHREAD
+** [[SQLITE_CONFIG_MULTITHREAD]]
SQLITE_CONFIG_MULTITHREAD
**
There are no arguments to this option. ^This option sets the ** [threading mode] to Multi-thread. In other words, it disables ** mutexing on [database connection] and [prepared statement] objects. @@ -1263,7 +1274,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] will return [SQLITE_ERROR] if called with the ** SQLITE_CONFIG_MULTITHREAD configuration option.
** -**
SQLITE_CONFIG_SERIALIZED
+** [[SQLITE_CONFIG_SERIALIZED]]
SQLITE_CONFIG_SERIALIZED
**
There are no arguments to this option. ^This option sets the ** [threading mode] to Serialized. In other words, this option enables ** all mutexes including the recursive @@ -1279,7 +1290,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] will return [SQLITE_ERROR] if called with the ** SQLITE_CONFIG_SERIALIZED configuration option.
** -**
SQLITE_CONFIG_MALLOC
+** [[SQLITE_CONFIG_MALLOC]]
SQLITE_CONFIG_MALLOC
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mem_methods] structure. The argument specifies ** alternative low-level memory allocation routines to be used in place of @@ -1287,7 +1298,7 @@ struct sqlite3_mem_methods { ** its own private copy of the content of the [sqlite3_mem_methods] structure ** before the [sqlite3_config()] call returns.
** -**
SQLITE_CONFIG_GETMALLOC
+** [[SQLITE_CONFIG_GETMALLOC]]
SQLITE_CONFIG_GETMALLOC
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ @@ -1295,7 +1306,7 @@ struct sqlite3_mem_methods { ** routines with a wrapper that simulations memory allocation failure or ** tracks memory usage, for example.
** -**
SQLITE_CONFIG_MEMSTATUS
+** [[SQLITE_CONFIG_MEMSTATUS]]
SQLITE_CONFIG_MEMSTATUS
**
^This option takes single argument of type int, interpreted as a ** boolean, which enables or disables the collection of memory allocation ** statistics. ^(When memory allocation statistics are disabled, the @@ -1311,7 +1322,7 @@ struct sqlite3_mem_methods { ** allocation statistics are disabled by default. **
** -**
SQLITE_CONFIG_SCRATCH
+** [[SQLITE_CONFIG_SCRATCH]]
SQLITE_CONFIG_SCRATCH
**
^This option specifies a static memory buffer that SQLite can use for ** scratch memory. There are three arguments: A pointer an 8-byte ** aligned memory buffer from which the scratch allocations will be @@ -1327,9 +1338,9 @@ struct sqlite3_mem_methods { ** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed.
** -**
SQLITE_CONFIG_PAGECACHE
+** [[SQLITE_CONFIG_PAGECACHE]]
SQLITE_CONFIG_PAGECACHE
**
^This option specifies a static memory buffer that SQLite can use for -** the database page cache with the default page cache implemenation. +** the database page cache with the default page cache implementation. ** This configuration should not be used if an application-define page ** cache implementation is loaded using the SQLITE_CONFIG_PCACHE option. ** There are three arguments to this option: A pointer to 8-byte aligned @@ -1348,7 +1359,7 @@ struct sqlite3_mem_methods { ** be aligned to an 8-byte boundary or subsequent behavior of SQLite ** will be undefined.
** -**
SQLITE_CONFIG_HEAP
+** [[SQLITE_CONFIG_HEAP]]
SQLITE_CONFIG_HEAP
**
^This option specifies a static memory buffer that SQLite will use ** for all of its dynamic memory allocation needs beyond those provided ** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. @@ -1365,7 +1376,7 @@ struct sqlite3_mem_methods { ** The minimum allocation size is capped at 2^12. Reasonable values ** for the minimum allocation size are 2^5 through 2^8.
** -**
SQLITE_CONFIG_MUTEX
+** [[SQLITE_CONFIG_MUTEX]]
SQLITE_CONFIG_MUTEX
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mutex_methods] structure. The argument specifies ** alternative low-level mutex routines to be used in place @@ -1377,7 +1388,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] with the SQLITE_CONFIG_MUTEX configuration option will ** return [SQLITE_ERROR].
** -**
SQLITE_CONFIG_GETMUTEX
+** [[SQLITE_CONFIG_GETMUTEX]]
SQLITE_CONFIG_GETMUTEX
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mutex_methods] structure. The ** [sqlite3_mutex_methods] @@ -1390,7 +1401,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] with the SQLITE_CONFIG_GETMUTEX configuration option will ** return [SQLITE_ERROR].
** -**
SQLITE_CONFIG_LOOKASIDE
+** [[SQLITE_CONFIG_LOOKASIDE]]
SQLITE_CONFIG_LOOKASIDE
**
^(This option takes two arguments that determine the default ** memory allocation for the lookaside memory allocator on each ** [database connection]. The first argument is the @@ -1400,18 +1411,18 @@ struct sqlite3_mem_methods { ** verb to [sqlite3_db_config()] can be used to change the lookaside ** configuration on individual connections.)^
** -**
SQLITE_CONFIG_PCACHE
+** [[SQLITE_CONFIG_PCACHE]]
SQLITE_CONFIG_PCACHE
**
^(This option takes a single argument which is a pointer to ** an [sqlite3_pcache_methods] object. This object specifies the interface ** to a custom page cache implementation.)^ ^SQLite makes a copy of the ** object and uses it for page cache memory allocations.
** -**
SQLITE_CONFIG_GETPCACHE
+** [[SQLITE_CONFIG_GETPCACHE]]
SQLITE_CONFIG_GETPCACHE
**
^(This option takes a single argument which is a pointer to an ** [sqlite3_pcache_methods] object. SQLite copies of the current ** page cache implementation into that object.)^
** -**
SQLITE_CONFIG_LOG
+** [[SQLITE_CONFIG_LOG]]
SQLITE_CONFIG_LOG
**
^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is @@ -1429,6 +1440,18 @@ struct sqlite3_mem_methods { ** In a multi-threaded application, the application-defined logger ** function must be threadsafe.
** +** [[SQLITE_CONFIG_URI]]
SQLITE_CONFIG_URI +**
This option takes a single argument of type int. If non-zero, then +** URI handling is globally enabled. If the parameter is zero, then URI handling +** is globally disabled. If URI handling is globally enabled, all filenames +** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +** specified as part of [ATTACH] commands are interpreted as URIs, regardless +** of whether or not the [SQLITE_OPEN_URI] flag is set when the database +** connection is opened. If it is globally disabled, filenames are +** only interpreted as URIs if the SQLITE_OPEN_URI flag is set when the +** database connection is opened. By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** [SQLITE_USE_URI] symbol defined. **
*/ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1447,6 +1470,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_PCACHE 14 /* sqlite3_pcache_methods* */ #define SQLITE_CONFIG_GETPCACHE 15 /* sqlite3_pcache_methods* */ #define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ /* ** CAPI3REF: Database Connection Configuration Options @@ -1532,13 +1556,17 @@ SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); ** ** ^This routine returns the [rowid] of the most recent ** successful [INSERT] into the database from the [database connection] -** in the first argument. ^If no successful [INSERT]s +** in the first argument. ^As of SQLite version 3.7.7, this routines +** records the last insert rowid of both ordinary tables and [virtual tables]. +** ^If no successful [INSERT]s ** have ever occurred on that database connection, zero is returned. ** -** ^(If an [INSERT] occurs within a trigger, then the [rowid] of the inserted -** row is returned by this routine as long as the trigger is running. -** But once the trigger terminates, the value returned by this routine -** reverts to the last value inserted before the trigger fired.)^ +** ^(If an [INSERT] occurs within a trigger or within a [virtual table] +** method, then this routine will return the [rowid] of the inserted +** row as long as the trigger or virtual table method is running. +** But once the trigger or virtual table method ends, the value returned +** by this routine reverts to what it was before the trigger or virtual +** table method began.)^ ** ** ^An [INSERT] that fails due to a constraint violation is not a ** successful [INSERT] and does not change the value returned by this @@ -2201,6 +2229,9 @@ SQLITE_API int sqlite3_set_authorizer( ** to signal SQLite whether or not the action is permitted. See the ** [sqlite3_set_authorizer | authorizer documentation] for additional ** information. +** +** Note that SQLITE_IGNORE is also used as a [SQLITE_ROLLBACK | return code] +** from the [sqlite3_vtab_on_conflict()] interface. */ #define SQLITE_DENY 1 /* Abort the SQL statement with an error */ #define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */ @@ -2323,7 +2354,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); /* ** CAPI3REF: Opening A New Database Connection ** -** ^These routines open an SQLite database file whose name is given by the +** ^These routines open an SQLite database file as specified by the ** filename argument. ^The filename argument is interpreted as UTF-8 for ** sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte ** order for sqlite3_open16(). ^(A [database connection] handle is usually @@ -2350,7 +2381,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** sqlite3_open_v2() can take one of ** the following three values, optionally combined with the ** [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], [SQLITE_OPEN_SHAREDCACHE], -** and/or [SQLITE_OPEN_PRIVATECACHE] flags:)^ +** [SQLITE_OPEN_PRIVATECACHE], and/or [SQLITE_OPEN_URI] flags:)^ ** **
** ^(
[SQLITE_OPEN_READONLY]
@@ -2369,9 +2400,8 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); **
** ** If the 3rd parameter to sqlite3_open_v2() is not one of the -** combinations shown above or one of the combinations shown above combined -** with the [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], -** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_PRIVATECACHE] flags, +** combinations shown above optionally combined with other +** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits] ** then the behavior is undefined. ** ** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection @@ -2386,6 +2416,11 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** [SQLITE_OPEN_PRIVATECACHE] flag causes the database connection to not ** participate in [shared cache mode] even if it is enabled. ** +** ^The fourth parameter to sqlite3_open_v2() is the name of the +** [sqlite3_vfs] object that defines the operating system interface that +** the new database connection should use. ^If the fourth parameter is +** a NULL pointer then the default [sqlite3_vfs] object is used. +** ** ^If the filename is ":memory:", then a private, temporary in-memory database ** is created for the connection. ^This in-memory database will vanish when ** the database connection is closed. Future versions of SQLite might @@ -2398,10 +2433,111 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** on-disk database will be created. ^This private database will be ** automatically deleted as soon as the database connection is closed. ** -** ^The fourth parameter to sqlite3_open_v2() is the name of the -** [sqlite3_vfs] object that defines the operating system interface that -** the new database connection should use. ^If the fourth parameter is -** a NULL pointer then the default [sqlite3_vfs] object is used. +** [[URI filenames in sqlite3_open()]]

URI Filenames

+** +** ^If [URI filename] interpretation is enabled, and the filename argument +** begins with "file:", then the filename is interpreted as a URI. ^URI +** filename interpretation is enabled if the [SQLITE_OPEN_URI] flag is +** set in the fourth argument to sqlite3_open_v2(), or if it has +** been enabled globally using the [SQLITE_CONFIG_URI] option with the +** [sqlite3_config()] method or by the [SQLITE_USE_URI] compile-time option. +** As of SQLite version 3.7.7, URI filename interpretation is turned off +** by default, but future releases of SQLite might enable URI filename +** interpretation by default. See "[URI filenames]" for additional +** information. +** +** URI filenames are parsed according to RFC 3986. ^If the URI contains an +** authority, then it must be either an empty string or the string +** "localhost". ^If the authority is not an empty string or "localhost", an +** error is returned to the caller. ^The fragment component of a URI, if +** present, is ignored. +** +** ^SQLite uses the path component of the URI as the name of the disk file +** which contains the database. ^If the path begins with a '/' character, +** then it is interpreted as an absolute path. ^If the path does not begin +** with a '/' (meaning that the authority section is omitted from the URI) +** then the path is interpreted as a relative path. +** ^On windows, the first component of an absolute path +** is a drive specification (e.g. "C:"). +** +** [[core URI query parameters]] +** The query component of a URI may contain parameters that are interpreted +** either by SQLite itself, or by a [VFS | custom VFS implementation]. +** SQLite interprets the following three query parameters: +** +**
    +**
  • vfs: ^The "vfs" parameter may be used to specify the name of +** a VFS object that provides the operating system interface that should +** be used to access the database file on disk. ^If this option is set to +** an empty string the default VFS object is used. ^Specifying an unknown +** VFS is an error. ^If sqlite3_open_v2() is used and the vfs option is +** present, then the VFS specified by the option takes precedence over +** the value passed as the fourth parameter to sqlite3_open_v2(). +** +**
  • mode: ^(The mode parameter may be set to either "ro", "rw" or +** "rwc". Attempting to set it to any other value is an error)^. +** ^If "ro" is specified, then the database is opened for read-only +** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the +** third argument to sqlite3_prepare_v2(). ^If the mode option is set to +** "rw", then the database is opened for read-write (but not create) +** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had +** been set. ^Value "rwc" is equivalent to setting both +** SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE. ^If sqlite3_open_v2() is +** used, it is an error to specify a value for the mode parameter that is +** less restrictive than that specified by the flags passed as the third +** parameter. +** +**
  • cache: ^The cache parameter may be set to either "shared" or +** "private". ^Setting it to "shared" is equivalent to setting the +** SQLITE_OPEN_SHAREDCACHE bit in the flags argument passed to +** sqlite3_open_v2(). ^Setting the cache parameter to "private" is +** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. +** ^If sqlite3_open_v2() is used and the "cache" parameter is present in +** a URI filename, its value overrides any behaviour requested by setting +** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +**
+** +** ^Specifying an unknown parameter in the query component of a URI is not an +** error. Future versions of SQLite might understand additional query +** parameters. See "[query parameters with special meaning to SQLite]" for +** additional information. +** +** [[URI filename examples]]

URI filename examples

+** +** +**
URI filenames Results +**
file:data.db +** Open the file "data.db" in the current directory. +**
file:/home/fred/data.db
+** file:///home/fred/data.db
+** file://localhost/home/fred/data.db
+** Open the database file "/home/fred/data.db". +**
file://darkstar/home/fred/data.db +** An error. "darkstar" is not a recognized authority. +**
+** file:///C:/Documents%20and%20Settings/fred/Desktop/data.db +** Windows only: Open the file "data.db" on fred's desktop on drive +** C:. Note that the %20 escaping in this example is not strictly +** necessary - space characters can be used literally +** in URI filenames. +**
file:data.db?mode=ro&cache=private +** Open file "data.db" in the current directory for read-only access. +** Regardless of whether or not shared-cache mode is enabled by +** default, use a private cache. +**
file:/home/fred/data.db?vfs=unix-nolock +** Open file "/home/fred/data.db". Use the special VFS "unix-nolock". +**
file:data.db?mode=readonly +** An error. "readonly" is not a valid option for the "mode" parameter. +**
+** +** ^URI hexadecimal escape sequences (%HH) are supported within the path and +** query components of a URI. A hexadecimal escape sequence consists of a +** percent sign - "%" - followed by exactly two hexadecimal digits +** specifying an octet value. ^Before the path or query components of a +** URI filename are interpreted, they are encoded using UTF-8 and all +** hexadecimal escape sequences replaced by a single byte containing the +** corresponding octet. If this process generates an invalid UTF-8 encoding, +** the results are undefined. ** ** Note to Windows users: The encoding used for the filename argument ** of sqlite3_open() and sqlite3_open_v2() must be UTF-8, not whatever @@ -2424,6 +2560,26 @@ SQLITE_API int sqlite3_open_v2( const char *zVfs /* Name of VFS module to use */ ); +/* +** CAPI3REF: Obtain Values For URI Parameters +** +** This is a utility routine, useful to VFS implementations, that checks +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of the query parameter. +** +** The zFilename argument is the filename pointer passed into the xOpen() +** method of a VFS implementation. The zParam argument is the name of the +** query parameter we seek. This routine returns the value of the zParam +** parameter if it exists. If the parameter does not exist, this routine +** returns a NULL pointer. +** +** If the zFilename argument to this function is not a pointer that SQLite +** passed into the xOpen VFS method, then the behavior of this routine +** is undefined and probably undesirable. +*/ +SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); + + /* ** CAPI3REF: Error Codes And Messages ** @@ -2539,43 +2695,45 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** Additional information is available at [limits | Limits in SQLite]. ** **
-** ^(
SQLITE_LIMIT_LENGTH
+** [[SQLITE_LIMIT_LENGTH]] ^(
SQLITE_LIMIT_LENGTH
**
The maximum size of any string or BLOB or table row, in bytes.
)^ ** -** ^(
SQLITE_LIMIT_SQL_LENGTH
+** [[SQLITE_LIMIT_SQL_LENGTH]] ^(
SQLITE_LIMIT_SQL_LENGTH
**
The maximum length of an SQL statement, in bytes.
)^ ** -** ^(
SQLITE_LIMIT_COLUMN
+** [[SQLITE_LIMIT_COLUMN]] ^(
SQLITE_LIMIT_COLUMN
**
The maximum number of columns in a table definition or in the ** result set of a [SELECT] or the maximum number of columns in an index ** or in an ORDER BY or GROUP BY clause.
)^ ** -** ^(
SQLITE_LIMIT_EXPR_DEPTH
+** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(
SQLITE_LIMIT_EXPR_DEPTH
**
The maximum depth of the parse tree on any expression.
)^ ** -** ^(
SQLITE_LIMIT_COMPOUND_SELECT
+** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(
SQLITE_LIMIT_COMPOUND_SELECT
**
The maximum number of terms in a compound SELECT statement.
)^ ** -** ^(
SQLITE_LIMIT_VDBE_OP
+** [[SQLITE_LIMIT_VDBE_OP]] ^(
SQLITE_LIMIT_VDBE_OP
**
The maximum number of instructions in a virtual machine program ** used to implement an SQL statement. This limit is not currently ** enforced, though that might be added in some future release of ** SQLite.
)^ ** -** ^(
SQLITE_LIMIT_FUNCTION_ARG
+** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
SQLITE_LIMIT_FUNCTION_ARG
**
The maximum number of arguments on a function.
)^ ** -** ^(
SQLITE_LIMIT_ATTACHED
+** [[SQLITE_LIMIT_ATTACHED]] ^(
SQLITE_LIMIT_ATTACHED
**
The maximum number of [ATTACH | attached databases].)^
** +** [[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]] ** ^(
SQLITE_LIMIT_LIKE_PATTERN_LENGTH
**
The maximum length of the pattern argument to the [LIKE] or ** [GLOB] operators.
)^ ** +** [[SQLITE_LIMIT_VARIABLE_NUMBER]] ** ^(
SQLITE_LIMIT_VARIABLE_NUMBER
**
The maximum index number of any [parameter] in an SQL statement.)^ ** -** ^(
SQLITE_LIMIT_TRIGGER_DEPTH
+** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
SQLITE_LIMIT_TRIGGER_DEPTH
**
The maximum depth of recursion for triggers.
)^ **
*/ @@ -3104,7 +3262,7 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** ^[SQLITE_BUSY] means that the database engine was unable to acquire the ** database locks it needs to do its job. ^If the statement is a [COMMIT] ** or occurs outside of an explicit transaction, then you can retry the -** statement. If the statement is not a [COMMIT] and occurs within a +** statement. If the statement is not a [COMMIT] and occurs within an ** explicit transaction then you should rollback the transaction before ** continuing. ** @@ -3383,7 +3541,7 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); ** CAPI3REF: Destroy A Prepared Statement Object ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. -** ^If the most recent evaluation of the statement encountered no errors or +** ^If the most recent evaluation of the statement encountered no errors ** or if the statement is never been evaluated, then sqlite3_finalize() returns ** SQLITE_OK. ^If the most recent evaluation of statement S failed, then ** sqlite3_finalize(S) returns the appropriate [error code] or @@ -4610,6 +4768,11 @@ struct sqlite3_module { void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), void **ppArg); int (*xRename)(sqlite3_vtab *pVtab, const char *zNew); + /* The methods above are in version 1 of the sqlite_module object. Those + ** below are for version 2 and greater. */ + int (*xSavepoint)(sqlite3_vtab *pVTab, int); + int (*xRelease)(sqlite3_vtab *pVTab, int); + int (*xRollbackTo)(sqlite3_vtab *pVTab, int); }; /* @@ -5292,7 +5455,7 @@ struct sqlite3_mutex_methods { ** ** ^If the argument to sqlite3_mutex_held() is a NULL pointer then ** the routine should return 1. This seems counter-intuitive since -** clearly the mutex cannot be held if it does not exist. But the +** clearly the mutex cannot be held if it does not exist. But ** the reason the mutex does not exist is because the build is not ** using mutexes. And we do not want the assert() containing the ** call to sqlite3_mutex_held() to fail, so a non-zero return is @@ -5415,7 +5578,8 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_PGHDRSZ 17 #define SQLITE_TESTCTRL_SCRATCHMALLOC 18 -#define SQLITE_TESTCTRL_LAST 18 +#define SQLITE_TESTCTRL_LOCALTIME_FAULT 19 +#define SQLITE_TESTCTRL_LAST 19 /* ** CAPI3REF: SQLite Runtime Status @@ -5424,7 +5588,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for ** the specific parameter to measure. ^(Recognized integer codes -** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...].)^ +** are of the form [status parameters | SQLITE_STATUS_...].)^ ** ^The current value of the parameter is returned into *pCurrent. ** ^The highest recorded value is returned in *pHighwater. ^If the ** resetFlag is true, then the highest record value is reset after @@ -5451,12 +5615,13 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF /* ** CAPI3REF: Status Parameters +** KEYWORDS: {status parameters} ** ** These integer constants designate various run-time status parameters ** that can be returned by [sqlite3_status()]. ** **
-** ^(
SQLITE_STATUS_MEMORY_USED
+** [[SQLITE_STATUS_MEMORY_USED]] ^(
SQLITE_STATUS_MEMORY_USED
**
This parameter is the current amount of memory checked out ** using [sqlite3_malloc()], either directly or indirectly. The ** figure includes calls made to [sqlite3_malloc()] by the application @@ -5466,23 +5631,24 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** this parameter. The amount returned is the sum of the allocation ** sizes as reported by the xSize method in [sqlite3_mem_methods].
)^ ** -** ^(
SQLITE_STATUS_MALLOC_SIZE
+** [[SQLITE_STATUS_MALLOC_SIZE]] ^(
SQLITE_STATUS_MALLOC_SIZE
**
This parameter records the largest memory allocation request ** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their ** internal equivalents). Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
)^ ** -** ^(
SQLITE_STATUS_MALLOC_COUNT
+** [[SQLITE_STATUS_MALLOC_COUNT]] ^(
SQLITE_STATUS_MALLOC_COUNT
**
This parameter records the number of separate memory allocations ** currently checked out.
)^ ** -** ^(
SQLITE_STATUS_PAGECACHE_USED
+** [[SQLITE_STATUS_PAGECACHE_USED]] ^(
SQLITE_STATUS_PAGECACHE_USED
**
This parameter returns the number of pages used out of the ** [pagecache memory allocator] that was configured using ** [SQLITE_CONFIG_PAGECACHE]. The ** value returned is in pages, not in bytes.
)^ ** +** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]] ** ^(
SQLITE_STATUS_PAGECACHE_OVERFLOW
**
This parameter returns the number of bytes of page cache ** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] @@ -5492,13 +5658,13 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because ** no space was left in the page cache.
)^ ** -** ^(
SQLITE_STATUS_PAGECACHE_SIZE
+** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(
SQLITE_STATUS_PAGECACHE_SIZE
**
This parameter records the largest memory allocation request ** handed to [pagecache memory allocator]. Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
)^ ** -** ^(
SQLITE_STATUS_SCRATCH_USED
+** [[SQLITE_STATUS_SCRATCH_USED]] ^(
SQLITE_STATUS_SCRATCH_USED
**
This parameter returns the number of allocations used out of the ** [scratch memory allocator] configured using ** [SQLITE_CONFIG_SCRATCH]. The value returned is in allocations, not @@ -5506,7 +5672,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** outstanding at time, this parameter also reports the number of threads ** using scratch memory at the same time.
)^ ** -** ^(
SQLITE_STATUS_SCRATCH_OVERFLOW
+** [[SQLITE_STATUS_SCRATCH_OVERFLOW]] ^(
SQLITE_STATUS_SCRATCH_OVERFLOW
**
This parameter returns the number of bytes of scratch memory ** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] ** buffer and where forced to overflow to [sqlite3_malloc()]. The values @@ -5516,13 +5682,13 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** slots were available. **
)^ ** -** ^(
SQLITE_STATUS_SCRATCH_SIZE
+** [[SQLITE_STATUS_SCRATCH_SIZE]] ^(
SQLITE_STATUS_SCRATCH_SIZE
**
This parameter records the largest memory allocation request ** handed to [scratch memory allocator]. Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
)^ ** -** ^(
SQLITE_STATUS_PARSER_STACK
+** [[SQLITE_STATUS_PARSER_STACK]] ^(
SQLITE_STATUS_PARSER_STACK
**
This parameter records the deepest parser stack. It is only ** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
)^ **
@@ -5547,9 +5713,9 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** about a single [database connection]. ^The first argument is the ** database connection object to be interrogated. ^The second argument ** is an integer constant, taken from the set of -** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros, that +** [SQLITE_DBSTATUS options], that ** determines the parameter to interrogate. The set of -** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros is likely +** [SQLITE_DBSTATUS options] is likely ** to grow in future releases of SQLite. ** ** ^The current value of the requested parameter is written into *pCur @@ -5566,6 +5732,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r /* ** CAPI3REF: Status Parameters for database connections +** KEYWORDS: {SQLITE_DBSTATUS options} ** ** These constants are the available integer "verbs" that can be passed as ** the second argument to the [sqlite3_db_status()] interface. @@ -5577,15 +5744,16 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** if a discontinued or unsupported verb is invoked. ** **
-** ^(
SQLITE_DBSTATUS_LOOKASIDE_USED
+** [[SQLITE_DBSTATUS_LOOKASIDE_USED]] ^(
SQLITE_DBSTATUS_LOOKASIDE_USED
**
This parameter returns the number of lookaside memory slots currently ** checked out.
)^ ** -** ^(
SQLITE_DBSTATUS_LOOKASIDE_HIT
+** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
SQLITE_DBSTATUS_LOOKASIDE_HIT
**
This parameter returns the number malloc attempts that were ** satisfied using lookaside memory. Only the high-water value is meaningful; ** the current value is always zero.)^ ** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]] ** ^(
SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE
**
This parameter returns the number malloc attempts that might have ** been satisfied using lookaside memory but failed due to the amount of @@ -5593,6 +5761,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** Only the high-water value is meaningful; ** the current value is always zero.)^ ** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]] ** ^(
SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL
**
This parameter returns the number malloc attempts that might have ** been satisfied using lookaside memory but failed due to all lookaside @@ -5600,12 +5769,12 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** Only the high-water value is meaningful; ** the current value is always zero.)^ ** -** ^(
SQLITE_DBSTATUS_CACHE_USED
+** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
SQLITE_DBSTATUS_CACHE_USED
**
This parameter returns the approximate number of of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. ** -** ^(
SQLITE_DBSTATUS_SCHEMA_USED
+** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
SQLITE_DBSTATUS_SCHEMA_USED
**
This parameter returns the approximate number of of bytes of heap ** memory used to store the schema for all databases associated ** with the connection - main, temp, and any [ATTACH]-ed databases.)^ @@ -5614,7 +5783,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** [shared cache mode] being enabled. ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. ** -** ^(
SQLITE_DBSTATUS_STMT_USED
+** [[SQLITE_DBSTATUS_STMT_USED]] ^(
SQLITE_DBSTATUS_STMT_USED
**
This parameter returns the approximate number of of bytes of heap ** and lookaside memory used by all prepared statements associated with ** the database connection.)^ @@ -5636,7 +5805,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** CAPI3REF: Prepared Statement Status ** ** ^(Each prepared statement maintains various -** [SQLITE_STMTSTATUS_SORT | counters] that measure the number +** [SQLITE_STMTSTATUS counters] that measure the number ** of times it has performed specific operations.)^ These counters can ** be used to monitor the performance characteristics of the prepared ** statements. For example, if the number of table steps greatly exceeds @@ -5647,7 +5816,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** ^(This interface is used to retrieve and reset counter values from ** a [prepared statement]. The first argument is the prepared statement ** object to be interrogated. The second argument -** is an integer code for a specific [SQLITE_STMTSTATUS_SORT | counter] +** is an integer code for a specific [SQLITE_STMTSTATUS counter] ** to be interrogated.)^ ** ^The current value of the requested counter is returned. ** ^If the resetFlg is true, then the counter is reset to zero after this @@ -5659,24 +5828,25 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); /* ** CAPI3REF: Status Parameters for prepared statements +** KEYWORDS: {SQLITE_STMTSTATUS counter} {SQLITE_STMTSTATUS counters} ** ** These preprocessor macros define integer codes that name counter ** values associated with the [sqlite3_stmt_status()] interface. ** The meanings of the various counters are as follows: ** **
-**
SQLITE_STMTSTATUS_FULLSCAN_STEP
+** [[SQLITE_STMTSTATUS_FULLSCAN_STEP]]
SQLITE_STMTSTATUS_FULLSCAN_STEP
**
^This is the number of times that SQLite has stepped forward in ** a table as part of a full table scan. Large numbers for this counter ** may indicate opportunities for performance improvement through ** careful use of indices.
** -**
SQLITE_STMTSTATUS_SORT
+** [[SQLITE_STMTSTATUS_SORT]]
SQLITE_STMTSTATUS_SORT
**
^This is the number of sort operations that have occurred. ** A non-zero value in this counter may indicate an opportunity to ** improvement performance through careful use of indices.
** -**
SQLITE_STMTSTATUS_AUTOINDEX
+** [[SQLITE_STMTSTATUS_AUTOINDEX]]
SQLITE_STMTSTATUS_AUTOINDEX
**
^This is the number of rows inserted into transient indices that ** were created automatically in order to help joins run faster. ** A non-zero value in this counter may indicate an opportunity to @@ -5727,6 +5897,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** the application may discard the parameter after the call to ** [sqlite3_config()] returns.)^ ** +** [[the xInit() page cache method]] ** ^(The xInit() method is called once for each effective ** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() @@ -5737,6 +5908,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** built-in default page cache is used instead of the application defined ** page cache.)^ ** +** [[the xShutdown() page cache method]] ** ^The xShutdown() method is called by [sqlite3_shutdown()]. ** It can be used to clean up ** any outstanding resources before process shutdown, if required. @@ -5751,6 +5923,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^SQLite will never invoke xInit() more than once without an intervening ** call to xShutdown(). ** +** [[the xCreate() page cache methods]] ** ^SQLite invokes the xCreate() method to construct a new cache instance. ** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The @@ -5775,6 +5948,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^Hence, a cache created with bPurgeable false will ** never contain any unpinned pages. ** +** [[the xCachesize() page cache method]] ** ^(The xCachesize() method may be called at any time by SQLite to set the ** suggested maximum cache-size (number of pages stored by) the cache ** instance passed as the first argument. This is the value configured using @@ -5782,14 +5956,16 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** parameter, the implementation is not required to do anything with this ** value; it is advisory only. ** +** [[the xPagecount() page cache methods]] ** The xPagecount() method must return the number of pages currently ** stored in the cache, both pinned and unpinned. ** +** [[the xFetch() page cache methods]] ** The xFetch() method locates a page in the cache and returns a pointer to ** the page, or a NULL pointer. ** A "page", in this context, means a buffer of szPage bytes aligned at an ** 8-byte boundary. The page to be fetched is determined by the key. ^The -** mimimum key value is 1. After it has been retrieved using xFetch, the page +** minimum key value is 1. After it has been retrieved using xFetch, the page ** is considered to be "pinned". ** ** If the requested page is already in the page cache, then the page cache @@ -5813,6 +5989,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** attempt to unpin one or more cache pages by spilling the content of ** pinned pages to disk and synching the operating system disk cache. ** +** [[the xUnpin() page cache method]] ** ^xUnpin() is called by SQLite with a pointer to a currently pinned page ** as its second argument. If the third parameter, discard, is non-zero, ** then the page must be evicted from the cache. @@ -5825,6 +6002,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** call to xUnpin() unpins the page regardless of the number of prior calls ** to xFetch(). ** +** [[the xRekey() page cache methods]] ** The xRekey() method is used to change the key value associated with the ** page passed as the second argument. If the cache ** previously contains an entry associated with newKey, it must be @@ -5837,6 +6015,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** of these pages are pinned, they are implicitly unpinned, meaning that ** they can be safely discarded. ** +** [[the xDestroy() page cache method]] ** ^The xDestroy() method is used to delete a cache allocated by xCreate(). ** All resources associated with the specified cache should be freed. ^After ** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*] @@ -5899,7 +6078,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** There should be exactly one call to sqlite3_backup_finish() for each ** successful call to sqlite3_backup_init(). ** -** sqlite3_backup_init() +** [[sqlite3_backup_init()]] sqlite3_backup_init() ** ** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the ** [database connection] associated with the destination database @@ -5926,7 +6105,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** sqlite3_backup_finish() functions to perform the specified backup ** operation. ** -** sqlite3_backup_step() +** [[sqlite3_backup_step()]] sqlite3_backup_step() ** ** ^Function sqlite3_backup_step(B,N) will copy up to N pages between ** the source and destination databases specified by [sqlite3_backup] object B. @@ -5983,7 +6162,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** by the backup operation, then the backup database is automatically ** updated at the same time. ** -** sqlite3_backup_finish() +** [[sqlite3_backup_finish()]] sqlite3_backup_finish() ** ** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the ** application wishes to abandon the backup operation, the application @@ -6006,7 +6185,8 @@ typedef struct sqlite3_backup sqlite3_backup; ** is not a permanent error and does not affect the return value of ** sqlite3_backup_finish(). ** -** sqlite3_backup_remaining(), sqlite3_backup_pagecount() +** [[sqlite3_backup__remaining()]] [[sqlite3_backup_pagecount()]] +** sqlite3_backup_remaining() and sqlite3_backup_pagecount() ** ** ^Each call to sqlite3_backup_step() sets two values inside ** the [sqlite3_backup] object: the number of pages still to be backed @@ -6392,6 +6572,93 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( #define SQLITE_CHECKPOINT_FULL 1 #define SQLITE_CHECKPOINT_RESTART 2 +/* +** CAPI3REF: Virtual Table Interface Configuration +** +** This function may be called by either the [xConnect] or [xCreate] method +** of a [virtual table] implementation to configure +** various facets of the virtual table interface. +** +** If this interface is invoked outside the context of an xConnect or +** xCreate virtual table method then the behavior is undefined. +** +** At present, there is only one option that may be configured using +** this function. (See [SQLITE_VTAB_CONSTRAINT_SUPPORT].) Further options +** may be added in the future. +*/ +SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Virtual Table Configuration Options +** +** These macros define the various options to the +** [sqlite3_vtab_config()] interface that [virtual table] implementations +** can use to customize and optimize their behavior. +** +**
+**
SQLITE_VTAB_CONSTRAINT_SUPPORT +**
Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported, +** where X is an integer. If X is zero, then the [virtual table] whose +** [xCreate] or [xConnect] method invoked [sqlite3_vtab_config()] does not +** support constraints. In this configuration (which is the default) if +** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire +** statement is rolled back as if [ON CONFLICT | OR ABORT] had been +** specified as part of the users SQL statement, regardless of the actual +** ON CONFLICT mode specified. +** +** If X is non-zero, then the virtual table implementation guarantees +** that if [xUpdate] returns [SQLITE_CONSTRAINT], it will do so before +** any modifications to internal or persistent data structures have been made. +** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite +** is able to roll back a statement or database transaction, and abandon +** or continue processing the current SQL statement as appropriate. +** If the ON CONFLICT mode is REPLACE and the [xUpdate] method returns +** [SQLITE_CONSTRAINT], SQLite handles this as if the ON CONFLICT mode +** had been ABORT. +** +** Virtual table implementations that are required to handle OR REPLACE +** must do so within the [xUpdate] method. If a call to the +** [sqlite3_vtab_on_conflict()] function indicates that the current ON +** CONFLICT policy is REPLACE, the virtual table implementation should +** silently replace the appropriate rows within the xUpdate callback and +** return SQLITE_OK. Or, if this is not possible, it may return +** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT +** constraint handling. +**
+*/ +#define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 + +/* +** CAPI3REF: Determine The Virtual Table Conflict Policy +** +** This function may only be called from within a call to the [xUpdate] method +** of a [virtual table] implementation for an INSERT or UPDATE operation. ^The +** value returned is one of [SQLITE_ROLLBACK], [SQLITE_IGNORE], [SQLITE_FAIL], +** [SQLITE_ABORT], or [SQLITE_REPLACE], according to the [ON CONFLICT] mode +** of the SQL statement that triggered the call to the [xUpdate] method of the +** [virtual table]. +*/ +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); + +/* +** CAPI3REF: Conflict resolution modes +** +** These constants are returned by [sqlite3_vtab_on_conflict()] to +** inform a [virtual table] implementation what the [ON CONFLICT] mode +** is for the SQL statement being evaluated. +** +** Note that the [SQLITE_IGNORE] constant is also used as a potential +** return value from the [sqlite3_set_authorizer()] callback and that +** [SQLITE_ABORT] is also a [result code]. +*/ +#define SQLITE_ROLLBACK 1 +/* #define SQLITE_IGNORE 2 // Also used by sqlite3_authorizer() callback */ +#define SQLITE_FAIL 3 +/* #define SQLITE_ABORT 4 // Also an error code */ +#define SQLITE_REPLACE 5 + + /* ** Undo the hack that converts floating point types to integer for diff --git a/harbour/contrib/3rd/sqlite3/sqlite3.hbp b/harbour/contrib/3rd/sqlite3/sqlite3.hbp index 91f860a165..604c34d42d 100644 --- a/harbour/contrib/3rd/sqlite3/sqlite3.hbp +++ b/harbour/contrib/3rd/sqlite3/sqlite3.hbp @@ -41,8 +41,8 @@ sqlite3.c # ORIGIN http://www.sqlite.org/ -# VER 3.7.6.3 -# URL http://www.sqlite.org/sqlite-amalgamation-3070603.zip +# VER 3.7.7.1 +# URL http://www.sqlite.org/sqlite-amalgamation-3070701.zip # DIFF sqlite3.dif # # MAP sqlite3.c diff --git a/harbour/src/3rd/png/LICENSE b/harbour/src/3rd/png/LICENSE index 6e4808cc0b..89f5243f16 100644 --- a/harbour/src/3rd/png/LICENSE +++ b/harbour/src/3rd/png/LICENSE @@ -10,7 +10,7 @@ this sentence. This code is released under the libpng license. -libpng versions 1.2.6, August 15, 2004, through 1.5.2, March 31, 2011, are +libpng versions 1.2.6, August 15, 2004, through 1.5.4, July 7, 2011, are Copyright (c) 2004, 2006-2011 Glenn Randers-Pehrson, and are distributed according to the same disclaimer and license as libpng-1.2.5 with the following individual added to the list of Contributing Authors @@ -108,4 +108,4 @@ certification mark of the Open Source Initiative. Glenn Randers-Pehrson glennrp at users.sourceforge.net -March 31, 2011 +July 7, 2011 diff --git a/harbour/src/3rd/png/Makefile b/harbour/src/3rd/png/Makefile index 2fa3cffe47..0ec1aaf67a 100644 --- a/harbour/src/3rd/png/Makefile +++ b/harbour/src/3rd/png/Makefile @@ -69,8 +69,8 @@ else endif # ORIGIN http://www.libpng.org/pub/png/libpng.html -# VER 1.5.2 -# URL http://prdownloads.sourceforge.net/libpng/libpng-1.5.2.tar.gz?download +# VER 1.5.4 +# URL http://prdownloads.sourceforge.net/libpng/libpng-1.5.4.tar.gz?download # DIFF png.dif # # MAP LICENSE diff --git a/harbour/src/3rd/png/png.c b/harbour/src/3rd/png/png.c index 7d09b0dbd2..eed3136195 100644 --- a/harbour/src/3rd/png/png.c +++ b/harbour/src/3rd/png/png.c @@ -1,7 +1,7 @@ /* png.c - location for general purpose libpng functions * - * Last changed in libpng 1.5.1 [February 3, 2011] + * Last changed in libpng 1.5.4 [July 7, 2011] * Copyright (c) 1998-2011 Glenn Randers-Pehrson * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) @@ -14,7 +14,7 @@ #include "pngpriv.h" /* Generate a compiler error if there is an old png.h in the search path. */ -typedef png_libpng_version_1_5_2 Your_png_h_is_not_version_1_5_2; +typedef png_libpng_version_1_5_4 Your_png_h_is_not_version_1_5_4; /* Tells libpng that we have already handled the first "num_bytes" bytes * of the PNG file signature. If the PNG data is embedded into another @@ -137,6 +137,61 @@ png_calculate_crc(png_structp png_ptr, png_const_bytep ptr, png_size_t length) png_ptr->crc = crc32(png_ptr->crc, ptr, (uInt)length); } +/* Check a user supplied version number, called from both read and write + * functions that create a png_struct + */ +int +png_user_version_check(png_structp png_ptr, png_const_charp user_png_ver) +{ + if (user_png_ver) + { + int i = 0; + + do + { + if (user_png_ver[i] != png_libpng_ver[i]) + png_ptr->flags |= PNG_FLAG_LIBRARY_MISMATCH; + } while (png_libpng_ver[i++]); + } + + else + png_ptr->flags |= PNG_FLAG_LIBRARY_MISMATCH; + + if (png_ptr->flags & PNG_FLAG_LIBRARY_MISMATCH) + { + /* Libpng 0.90 and later are binary incompatible with libpng 0.89, so + * we must recompile any applications that use any older library version. + * For versions after libpng 1.0, we will be compatible, so we need + * only check the first digit. + */ + if (user_png_ver == NULL || user_png_ver[0] != png_libpng_ver[0] || + (user_png_ver[0] == '1' && user_png_ver[2] != png_libpng_ver[2]) || + (user_png_ver[0] == '0' && user_png_ver[2] < '9')) + { +#ifdef PNG_WARNINGS_SUPPORTED + size_t pos = 0; + char m[128]; + + pos = png_safecat(m, sizeof m, pos, "Application built with libpng-"); + pos = png_safecat(m, sizeof m, pos, user_png_ver); + pos = png_safecat(m, sizeof m, pos, " but running with "); + pos = png_safecat(m, sizeof m, pos, png_libpng_ver); + + png_warning(png_ptr, m); +#endif + +#ifdef PNG_ERROR_NUMBERS_SUPPORTED + png_ptr->flags = 0; +#endif + + return 0; + } + } + + /* Success return. */ + return 1; +} + /* Allocate the memory for an info_struct for the application. We don't * really need the png_ptr, but it could potentially be useful in the * future. This should be used in favour of malloc(png_sizeof(png_info)) @@ -291,12 +346,10 @@ png_free_data(png_structp png_ptr, png_infop info_ptr, png_uint_32 mask, /* Free any sCAL entry */ if ((mask & PNG_FREE_SCAL) & info_ptr->free_me) { -#if defined(PNG_FIXED_POINT_SUPPORTED) && !defined(PNG_FLOATING_POINT_SUPPORTED) png_free(png_ptr, info_ptr->scal_s_width); png_free(png_ptr, info_ptr->scal_s_height); info_ptr->scal_s_width = NULL; info_ptr->scal_s_height = NULL; -#endif info_ptr->valid &= ~PNG_INFO_sCAL; } #endif @@ -518,28 +571,37 @@ png_convert_to_rfc1123(png_structp png_ptr, png_const_timep ptime) if (png_ptr == NULL) return (NULL); - if (png_ptr->time_buffer == NULL) { - png_ptr->time_buffer = (png_charp)png_malloc(png_ptr, (png_uint_32)(29* - png_sizeof(char))); + size_t pos = 0; + char number_buf[5]; /* enough for a four digit year */ + +# define APPEND_STRING(string)\ + pos = png_safecat(png_ptr->time_buffer, sizeof png_ptr->time_buffer,\ + pos, (string)) +# define APPEND_NUMBER(format, value)\ + APPEND_STRING(PNG_FORMAT_NUMBER(number_buf, format, (value))) +# define APPEND(ch)\ + if (pos < (sizeof png_ptr->time_buffer)-1)\ + png_ptr->time_buffer[pos++] = (ch) + + APPEND_NUMBER(PNG_NUMBER_FORMAT_u, (unsigned)ptime->day % 32); + APPEND(' '); + APPEND_STRING(short_months[(ptime->month - 1) % 12]); + APPEND(' '); + APPEND_NUMBER(PNG_NUMBER_FORMAT_u, ptime->year); + APPEND(' '); + APPEND_NUMBER(PNG_NUMBER_FORMAT_02u, (unsigned)ptime->hour % 24); + APPEND(':'); + APPEND_NUMBER(PNG_NUMBER_FORMAT_02u, (unsigned)ptime->minute % 60); + APPEND(':'); + APPEND_NUMBER(PNG_NUMBER_FORMAT_02u, (unsigned)ptime->second % 61); + APPEND_STRING(" +0000"); /* This reliably terminates the buffer */ + +# undef APPEND +# undef APPEND_NUMBER +# undef APPEND_STRING } -# ifdef USE_FAR_KEYWORD - { - char near_time_buf[29]; - png_snprintf6(near_time_buf, 29, "%d %s %d %02d:%02d:%02d +0000", - ptime->day % 32, short_months[(ptime->month - 1) % 12], - ptime->year, ptime->hour % 24, ptime->minute % 60, - ptime->second % 61); - png_memcpy(png_ptr->time_buffer, near_time_buf, - 29*png_sizeof(char)); - } -# else - png_snprintf6(png_ptr->time_buffer, 29, "%d %s %d %02d:%02d:%02d +0000", - ptime->day % 32, short_months[(ptime->month - 1) % 12], - ptime->year, ptime->hour % 24, ptime->minute % 60, - ptime->second % 61); -# endif return png_ptr->time_buffer; } # endif /* PNG_TIME_RFC1123_SUPPORTED */ @@ -555,13 +617,13 @@ png_get_copyright(png_const_structp png_ptr) #else # ifdef __STDC__ return PNG_STRING_NEWLINE \ - "libpng version 1.5.2 - March 31, 2011" PNG_STRING_NEWLINE \ + "libpng version 1.5.4 - July 7, 2011" PNG_STRING_NEWLINE \ "Copyright (c) 1998-2011 Glenn Randers-Pehrson" PNG_STRING_NEWLINE \ "Copyright (c) 1996-1997 Andreas Dilger" PNG_STRING_NEWLINE \ "Copyright (c) 1995-1996 Guy Eric Schalnat, Group 42, Inc." \ PNG_STRING_NEWLINE; # else - return "libpng version 1.5.2 - March 31, 2011\ + return "libpng version 1.5.4 - July 7, 2011\ Copyright (c) 1998-2011 Glenn Randers-Pehrson\ Copyright (c) 1996-1997 Andreas Dilger\ Copyright (c) 1995-1996 Guy Eric Schalnat, Group 42, Inc."; @@ -681,6 +743,13 @@ png_check_cHRM_fixed(png_structp png_ptr, if (png_ptr == NULL) return 0; + /* (x,y,z) values are first limited to 0..100000 (PNG_FP_1), the white + * y must also be greater than 0. To test for the upper limit calculate + * (PNG_FP_1-y) - x must be <= to this for z to be >= 0 (and the expression + * cannot overflow.) At this point we know x and y are >= 0 and (x+y) is + * <= PNG_FP_1. The previous test on PNG_MAX_UINT_31 is removed because it + * pointless (and it produces compiler warnings!) + */ if (white_x < 0 || white_y <= 0 || red_x < 0 || red_y < 0 || green_x < 0 || green_y < 0 || @@ -690,38 +759,26 @@ png_check_cHRM_fixed(png_structp png_ptr, "Ignoring attempt to set negative chromaticity value"); ret = 0; } - if (white_x > (png_fixed_point)PNG_UINT_31_MAX || - white_y > (png_fixed_point)PNG_UINT_31_MAX || - red_x > (png_fixed_point)PNG_UINT_31_MAX || - red_y > (png_fixed_point)PNG_UINT_31_MAX || - green_x > (png_fixed_point)PNG_UINT_31_MAX || - green_y > (png_fixed_point)PNG_UINT_31_MAX || - blue_x > (png_fixed_point)PNG_UINT_31_MAX || - blue_y > (png_fixed_point)PNG_UINT_31_MAX ) - { - png_warning(png_ptr, - "Ignoring attempt to set chromaticity value exceeding 21474.83"); - ret = 0; - } - if (white_x > 100000L - white_y) + /* And (x+y) must be <= PNG_FP_1 (so z is >= 0) */ + if (white_x > PNG_FP_1 - white_y) { png_warning(png_ptr, "Invalid cHRM white point"); ret = 0; } - if (red_x > 100000L - red_y) + if (red_x > PNG_FP_1 - red_y) { png_warning(png_ptr, "Invalid cHRM red point"); ret = 0; } - if (green_x > 100000L - green_y) + if (green_x > PNG_FP_1 - green_y) { png_warning(png_ptr, "Invalid cHRM green point"); ret = 0; } - if (blue_x > 100000L - blue_y) + if (blue_x > PNG_FP_1 - blue_y) { png_warning(png_ptr, "Invalid cHRM blue point"); ret = 0; @@ -763,7 +820,7 @@ png_check_IHDR(png_structp png_ptr, } # ifdef PNG_SET_USER_LIMITS_SUPPORTED - if (width > png_ptr->user_width_max || width > PNG_USER_WIDTH_MAX) + if (width > png_ptr->user_width_max) # else if (width > PNG_USER_WIDTH_MAX) @@ -774,7 +831,7 @@ png_check_IHDR(png_structp png_ptr, } # ifdef PNG_SET_USER_LIMITS_SUPPORTED - if (height > png_ptr->user_height_max || height > PNG_USER_HEIGHT_MAX) + if (height > png_ptr->user_height_max) # else if (height > PNG_USER_HEIGHT_MAX) # endif @@ -889,16 +946,9 @@ png_check_IHDR(png_structp png_ptr, /* Check an ASCII formated floating point value, see the more detailed * comments in pngpriv.h */ -/* The following is used internally to preserve the 'valid' flag */ +/* The following is used internally to preserve the sticky flags */ #define png_fp_add(state, flags) ((state) |= (flags)) -#define png_fp_set(state, value)\ - ((state) = (value) | ((state) & PNG_FP_WAS_VALID)) - -/* Internal type codes: bits above the base state! */ -#define PNG_FP_SIGN 0 /* [+-] */ -#define PNG_FP_DOT 4 /* . */ -#define PNG_FP_DIGIT 8 /* [0123456789] */ -#define PNG_FP_E 12 /* [Ee] */ +#define png_fp_set(state, value) ((state) = (value) | ((state) & PNG_FP_STICKY)) int /* PRIVATE */ png_check_fp_number(png_const_charp string, png_size_t size, int *statep, @@ -911,55 +961,55 @@ png_check_fp_number(png_const_charp string, png_size_t size, int *statep, { int type; /* First find the type of the next character */ + switch (string[i]) { - char ch = string[i]; - - if (ch >= 48 && ch <= 57) - type = PNG_FP_DIGIT; - - else switch (ch) - { - case 43: case 45: type = PNG_FP_SIGN; break; - case 46: type = PNG_FP_DOT; break; - case 69: case 101: type = PNG_FP_E; break; - default: goto PNG_FP_End; - } + case 43: type = PNG_FP_SAW_SIGN; break; + case 45: type = PNG_FP_SAW_SIGN + PNG_FP_NEGATIVE; break; + case 46: type = PNG_FP_SAW_DOT; break; + case 48: type = PNG_FP_SAW_DIGIT; break; + case 49: case 50: case 51: case 52: + case 53: case 54: case 55: case 56: + case 57: type = PNG_FP_SAW_DIGIT + PNG_FP_NONZERO; break; + case 69: + case 101: type = PNG_FP_SAW_E; break; + default: goto PNG_FP_End; } /* Now deal with this type according to the current * state, the type is arranged to not overlap the * bits of the PNG_FP_STATE. */ - switch ((state & PNG_FP_STATE) + type) + switch ((state & PNG_FP_STATE) + (type & PNG_FP_SAW_ANY)) { - case PNG_FP_INTEGER + PNG_FP_SIGN: + case PNG_FP_INTEGER + PNG_FP_SAW_SIGN: if (state & PNG_FP_SAW_ANY) goto PNG_FP_End; /* not a part of the number */ - png_fp_add(state, PNG_FP_SAW_SIGN); + png_fp_add(state, type); break; - case PNG_FP_INTEGER + PNG_FP_DOT: + case PNG_FP_INTEGER + PNG_FP_SAW_DOT: /* Ok as trailer, ok as lead of fraction. */ if (state & PNG_FP_SAW_DOT) /* two dots */ goto PNG_FP_End; else if (state & PNG_FP_SAW_DIGIT) /* trailing dot? */ - png_fp_add(state, PNG_FP_SAW_DOT); + png_fp_add(state, type); else - png_fp_set(state, PNG_FP_FRACTION | PNG_FP_SAW_DOT); + png_fp_set(state, PNG_FP_FRACTION | type); break; - case PNG_FP_INTEGER + PNG_FP_DIGIT: + case PNG_FP_INTEGER + PNG_FP_SAW_DIGIT: if (state & PNG_FP_SAW_DOT) /* delayed fraction */ png_fp_set(state, PNG_FP_FRACTION | PNG_FP_SAW_DOT); - png_fp_add(state, PNG_FP_SAW_DIGIT + PNG_FP_WAS_VALID); + png_fp_add(state, type | PNG_FP_WAS_VALID); break; - case PNG_FP_INTEGER + PNG_FP_E: + + case PNG_FP_INTEGER + PNG_FP_SAW_E: if ((state & PNG_FP_SAW_DIGIT) == 0) goto PNG_FP_End; @@ -967,17 +1017,17 @@ png_check_fp_number(png_const_charp string, png_size_t size, int *statep, break; - /* case PNG_FP_FRACTION + PNG_FP_SIGN: - goto PNG_FP_End; ** no sign in exponent */ + /* case PNG_FP_FRACTION + PNG_FP_SAW_SIGN: + goto PNG_FP_End; ** no sign in fraction */ - /* case PNG_FP_FRACTION + PNG_FP_DOT: + /* case PNG_FP_FRACTION + PNG_FP_SAW_DOT: goto PNG_FP_End; ** Because SAW_DOT is always set */ - case PNG_FP_FRACTION + PNG_FP_DIGIT: - png_fp_add(state, PNG_FP_SAW_DIGIT + PNG_FP_WAS_VALID); + case PNG_FP_FRACTION + PNG_FP_SAW_DIGIT: + png_fp_add(state, type | PNG_FP_WAS_VALID); break; - case PNG_FP_FRACTION + PNG_FP_E: + case PNG_FP_FRACTION + PNG_FP_SAW_E: /* This is correct because the trailing '.' on an * integer is handled above - so we can only get here * with the sequence ".E" (with no preceding digits). @@ -989,7 +1039,7 @@ png_check_fp_number(png_const_charp string, png_size_t size, int *statep, break; - case PNG_FP_EXPONENT + PNG_FP_SIGN: + case PNG_FP_EXPONENT + PNG_FP_SAW_SIGN: if (state & PNG_FP_SAW_ANY) goto PNG_FP_End; /* not a part of the number */ @@ -997,15 +1047,15 @@ png_check_fp_number(png_const_charp string, png_size_t size, int *statep, break; - /* case PNG_FP_EXPONENT + PNG_FP_DOT: + /* case PNG_FP_EXPONENT + PNG_FP_SAW_DOT: goto PNG_FP_End; */ - case PNG_FP_EXPONENT + PNG_FP_DIGIT: - png_fp_add(state, PNG_FP_SAW_DIGIT + PNG_FP_WAS_VALID); + case PNG_FP_EXPONENT + PNG_FP_SAW_DIGIT: + png_fp_add(state, PNG_FP_SAW_DIGIT | PNG_FP_WAS_VALID); break; - /* case PNG_FP_EXPONEXT + PNG_FP_E: + /* case PNG_FP_EXPONEXT + PNG_FP_SAW_E: goto PNG_FP_End; */ default: goto PNG_FP_End; /* I.e. break 2 */ @@ -1033,8 +1083,11 @@ png_check_fp_string(png_const_charp string, png_size_t size) int state=0; png_size_t char_index=0; - return png_check_fp_number(string, size, &state, &char_index) && - (char_index == size || string[char_index] == 0); + if (png_check_fp_number(string, size, &state, &char_index) && + (char_index == size || string[char_index] == 0)) + return state /* must be non-zero - see above */; + + return 0; /* i.e. fail */ } #endif /* pCAL or sCAL */ @@ -1102,7 +1155,7 @@ png_ascii_from_fp(png_structp png_ptr, png_charp ascii, png_size_t size, if (fp < 0) { fp = -fp; - *ascii++ = 45; /* '-' PLUS 1 TOTAL 1*/ + *ascii++ = 45; /* '-' PLUS 1 TOTAL 1 */ --size; } @@ -1329,7 +1382,7 @@ png_ascii_from_fp(png_structp png_ptr, png_charp ascii, png_size_t size, */ size -= cdigits; - *ascii++ = 69, --size; /* 'E': PLUS 1 TOTAL 2+precision*/ + *ascii++ = 69, --size; /* 'E': PLUS 1 TOTAL 2+precision */ if (exp_b10 < 0) { *ascii++ = 45, --size; /* '-': PLUS 1 TOTAL 3+precision */ @@ -1401,7 +1454,7 @@ png_ascii_from_fixed(png_structp png_ptr, png_charp ascii, png_size_t size, if (num <= 0x80000000U) /* else overflowed */ { - unsigned int ndigits = 0, first = 16/*flag value*/; + unsigned int ndigits = 0, first = 16 /* flag value */; char digits[10]; while (num) @@ -1495,7 +1548,7 @@ png_muldiv(png_fixed_point_p res, png_fixed_point a, png_int_32 times, r /= divisor; r = floor(r+.5); - /* A png_fixed_point is a 32 bit integer. */ + /* A png_fixed_point is a 32-bit integer. */ if (r <= 2147483647. && r >= -2147483648.) { *res = (png_fixed_point)r; @@ -1540,7 +1593,7 @@ png_muldiv(png_fixed_point_p res, png_fixed_point a, png_int_32 times, if (s32 < D) /* else overflow */ { - /* s32.s00 is now the 64 bit product, do a standard + /* s32.s00 is now the 64-bit product, do a standard * division, we know that s32 < D, so the maximum * required shift is 31. */ @@ -1683,7 +1736,7 @@ png_reciprocal2(png_fixed_point a, png_fixed_point b) * 2010: moved from pngset.c) */ /* * Multiply two 32-bit numbers, V1 and V2, using 32-bit - * arithmetic, to produce a 64 bit result in the HI/LO words. + * arithmetic, to produce a 64-bit result in the HI/LO words. * * A B * x C D @@ -1727,17 +1780,17 @@ png_64bit_product (long v1, long v2, unsigned long *hi_product, /* Fixed point gamma. * * To calculate gamma this code implements fast log() and exp() calls using only - * fixed point arithmetic. This code has sufficient precision for either 8 or - * 16 bit sample values. + * fixed point arithmetic. This code has sufficient precision for either 8-bit + * or 16-bit sample values. * * The tables used here were calculated using simple 'bc' programs, but C double * precision floating point arithmetic would work fine. The programs are given * at the head of each table. * - * 8 bit log table + * 8-bit log table * This is a table of -log(value/255)/log(2) for 'value' in the range 128 to - * 255, so it's the base 2 logarithm of a normalized 8 bit floating point - * mantissa. The numbers are 32 bit fractions. + * 255, so it's the base 2 logarithm of a normalized 8-bit floating point + * mantissa. The numbers are 32-bit fractions. */ static png_uint_32 png_8bit_l2[128] = @@ -1768,10 +1821,10 @@ png_8bit_l2[128] = 172473545U, 147538590U, 122703574U, 97967701U, 73330182U, 48790236U, 24347096U, 0U #if 0 - /* The following are the values for 16 bit tables - these work fine for the 8 - * bit conversions but produce very slightly larger errors in the 16 bit log - * (about 1.2 as opposed to 0.7 absolute error in the final value). To use - * these all the shifts below must be adjusted appropriately. + /* The following are the values for 16-bit tables - these work fine for the + * 8-bit conversions but produce very slightly larger errors in the 16-bit + * log (about 1.2 as opposed to 0.7 absolute error in the final value). To + * use these all the shifts below must be adjusted appropriately. */ 65166, 64430, 63700, 62976, 62257, 61543, 60835, 60132, 59434, 58741, 58054, 57371, 56693, 56020, 55352, 54689, 54030, 53375, 52726, 52080, 51439, 50803, @@ -1788,7 +1841,7 @@ png_8bit_l2[128] = #endif }; -static png_int_32 +PNG_STATIC png_int_32 png_log8bit(unsigned int x) { unsigned int lg2 = 0; @@ -1814,11 +1867,11 @@ png_log8bit(unsigned int x) return (png_int_32)((lg2 << 16) + ((png_8bit_l2[x-128]+32768)>>16)); } -/* The above gives exact (to 16 binary places) log2 values for 8 bit images, - * for 16 bit images we use the most significant 8 bits of the 16 bit value to +/* The above gives exact (to 16 binary places) log2 values for 8-bit images, + * for 16-bit images we use the most significant 8 bits of the 16-bit value to * get an approximation then multiply the approximation by a correction factor * determined by the remaining up to 8 bits. This requires an additional step - * in the 16 bit case. + * in the 16-bit case. * * We want log2(value/65535), we have log2(v'/255), where: * @@ -1827,8 +1880,8 @@ png_log8bit(unsigned int x) * * So f is value/v', which is equal to (256+v''/v') since v' is in the range 128 * to 255 and v'' is in the range 0 to 255 f will be in the range 256 to less - * than 258. The final factor also needs to correct for the fact that our 8 bit - * value is scaled by 255, whereas the 16 bit values must be scaled by 65535. + * than 258. The final factor also needs to correct for the fact that our 8-bit + * value is scaled by 255, whereas the 16-bit values must be scaled by 65535. * * This gives a final formula using a calculated value 'x' which is value/v' and * scaling by 65536 to match the above table: @@ -1838,13 +1891,13 @@ png_log8bit(unsigned int x) * Since these numbers are so close to '1' we can use simple linear * interpolation between the two end values 256/257 (result -368.61) and 258/257 * (result 367.179). The values used below are scaled by a further 64 to give - * 16 bit precision in the interpolation: + * 16-bit precision in the interpolation: * * Start (256): -23591 * Zero (257): 0 * End (258): 23499 */ -static png_int_32 +PNG_STATIC png_int_32 png_log16bit(png_uint_32 x) { unsigned int lg2 = 0; @@ -1865,7 +1918,7 @@ png_log16bit(png_uint_32 x) if ((x & 0x8000) == 0) lg2 += 1, x <<= 1; - /* Calculate the base logarithm from the top 8 bits as a 28 bit fractional + /* Calculate the base logarithm from the top 8 bits as a 28-bit fractional * value. */ lg2 <<= 28; @@ -1895,18 +1948,18 @@ png_log16bit(png_uint_32 x) return (png_int_32)((lg2 + 2048) >> 12); } -/* The 'exp()' case must invert the above, taking a 20 bit fixed point - * logarithmic value and returning a 16 or 8 bit number as appropriate. In +/* The 'exp()' case must invert the above, taking a 20-bit fixed point + * logarithmic value and returning a 16 or 8-bit number as appropriate. In * each case only the low 16 bits are relevant - the fraction - since the * integer bits (the top 4) simply determine a shift. * - * The worst case is the 16 bit distinction between 65535 and 65534, this + * The worst case is the 16-bit distinction between 65535 and 65534, this * requires perhaps spurious accuracty in the decoding of the logarithm to * distinguish log2(65535/65534.5) - 10^-5 or 17 bits. There is little chance * of getting this accuracy in practice. * * To deal with this the following exp() function works out the exponent of the - * frational part of the logarithm by using an accurate 32 bit value from the + * frational part of the logarithm by using an accurate 32-bit value from the * top four fractional bits then multiplying in the remaining bits. */ static png_uint_32 @@ -1915,7 +1968,7 @@ png_32bit_exp[16] = # if PNG_DO_BC for (i=0;i<16;++i) { .5 + e(-i/16*l(2))*2^32; } # endif - /* NOTE: the first entry is deliberately set to the maximum 32 bit value. */ + /* NOTE: the first entry is deliberately set to the maximum 32-bit value. */ 4294967295U, 4112874773U, 3938502376U, 3771522796U, 3611622603U, 3458501653U, 3311872529U, 3171459999U, 3037000500U, 2908241642U, 2784941738U, 2666869345U, 2553802834U, 2445529972U, 2341847524U, 2242560872U @@ -1938,12 +1991,12 @@ for (i=11;i>=0;--i){ print i, " ", (1 - e(-(2^i)/65536*l(2))) * 2^(32-i), "\n"} 0 45425.85339951654943850496 #endif -static png_uint_32 +PNG_STATIC png_uint_32 png_exp(png_fixed_point x) { if (x > 0 && x <= 0xfffff) /* Else overflow or zero (underflow) */ { - /* Obtain a 4 bit approximation */ + /* Obtain a 4-bit approximation */ png_uint_32 e = png_32bit_exp[(x >> 12) & 0xf]; /* Incorporate the low 12 bits - these decrease the returned value by @@ -1986,13 +2039,13 @@ png_exp(png_fixed_point x) return 0; } -static png_byte +PNG_STATIC png_byte png_exp8bit(png_fixed_point lg2) { - /* Get a 32 bit value: */ + /* Get a 32-bit value: */ png_uint_32 x = png_exp(lg2); - /* Convert the 32 bit value to 0..255 by multiplying by 256-1, note that the + /* Convert the 32-bit value to 0..255 by multiplying by 256-1, note that the * second, rounding, step can't overflow because of the first, subtraction, * step. */ @@ -2000,13 +2053,13 @@ png_exp8bit(png_fixed_point lg2) return (png_byte)((x + 0x7fffffU) >> 24); } -static png_uint_16 +PNG_STATIC png_uint_16 png_exp16bit(png_fixed_point lg2) { - /* Get a 32 bit value: */ + /* Get a 32-bit value: */ png_uint_32 x = png_exp(lg2); - /* Convert the 32 bit value to 0..65535 by multiplying by 65536-1: */ + /* Convert the 32-bit value to 0..65535 by multiplying by 65536-1: */ x -= x >> 16; return (png_uint_16)((x + 32767U) >> 16); } @@ -2059,9 +2112,9 @@ png_gamma_16bit_correct(unsigned int value, png_fixed_point gamma_val) } /* This does the right thing based on the bit_depth field of the - * png_struct, interpreting values as 8 or 16 bit. While the result - * is nominally a 16 bit value if bit depth is 8 then the result is - * 8 bit (as are the arguments.) + * png_struct, interpreting values as 8-bit or 16-bit. While the result + * is nominally a 16-bit value if bit depth is 8 then the result is + * 8-bit (as are the arguments.) */ png_uint_16 /* PRIVATE */ png_gamma_correct(png_structp png_ptr, unsigned int value, @@ -2084,7 +2137,7 @@ png_gamma_significant(png_fixed_point gamma_val) gamma_val > PNG_FP_1 + PNG_GAMMA_THRESHOLD_FIXED; } -/* Internal function to build a single 16 bit table - the table consists of +/* Internal function to build a single 16-bit table - the table consists of * 'num' 256 entry subtables, where 'num' is determined by 'shift' - the amount * to shift the input values right (or 16-number_of_signifiant_bits). * @@ -2111,7 +2164,7 @@ png_build_16bit_table(png_structp png_ptr, png_uint_16pp *ptable, (png_uint_16p)png_malloc(png_ptr, 256 * png_sizeof(png_uint_16)); /* The 'threshold' test is repeated here because it can arise for one of - * the 16 bit tables even if the others don't hit it. + * the 16-bit tables even if the others don't hit it. */ if (png_gamma_significant(gamma_val)) { @@ -2173,7 +2226,7 @@ png_build_16to8_table(png_structp png_ptr, png_uint_16pp *ptable, (png_uint_16pp)png_calloc(png_ptr, num * png_sizeof(png_uint_16p)); /* 'num' is the number of tables and also the number of low bits of low - * bits of the input 16 bit value used to select a table. Each table is + * bits of the input 16-bit value used to select a table. Each table is * itself index by the high 8 bits of the value. */ for (i = 0; i < num; i++) @@ -2183,24 +2236,24 @@ png_build_16to8_table(png_structp png_ptr, png_uint_16pp *ptable, /* 'gamma_val' is set to the reciprocal of the value calculated above, so * pow(out,g) is an *input* value. 'last' is the last input value set. * - * In the loop 'i' is used to find output values. Since the output is 8 - * bit there are only 256 possible values. The tables are set up to + * In the loop 'i' is used to find output values. Since the output is + * 8-bit there are only 256 possible values. The tables are set up to * select the closest possible output value for each input by finding * the input value at the boundary between each pair of output values * and filling the table up to that boundary with the lower output * value. * - * The boundary values are 0.5,1.5..253.5,254.5. Since these are 9 bit - * values the code below uses a 16 bit value in i; the values start at + * The boundary values are 0.5,1.5..253.5,254.5. Since these are 9-bit + * values the code below uses a 16-bit value in i; the values start at * 128.5 (for 0.5) and step by 257, for a total of 254 values (the last * entries are filled with 255). Start i at 128 and fill all 'last' * table entries <= 'max' */ last = 0; - for (i = 0; i < 255; ++i) /* 8 bit output value */ + for (i = 0; i < 255; ++i) /* 8-bit output value */ { /* Find the corresponding maximum input value */ - png_uint_16 out = (png_uint_16)(i * 257U); /* 16 bit output value */ + png_uint_16 out = (png_uint_16)(i * 257U); /* 16-bit output value */ /* Find the boundary value in 16 bits: */ png_uint_32 bound = png_gamma_16bit_correct(out+128U, gamma_val); @@ -2223,7 +2276,7 @@ png_build_16to8_table(png_structp png_ptr, png_uint_16pp *ptable, } } -/* Build a single 8 bit table: same as the 16 bit case but much simpler (and +/* Build a single 8-bit table: same as the 16-bit case but much simpler (and * typically much faster). Note that libpng currently does no sBIT processing * (apparently contrary to the spec) so a 256 entry table is always generated. */ @@ -2258,8 +2311,9 @@ png_build_gamma_table(png_structp png_ptr, int bit_depth) png_ptr->screen_gamma) : PNG_FP_1); #if defined(PNG_READ_BACKGROUND_SUPPORTED) || \ + defined(PNG_READ_ALPHA_MODE_SUPPORTED) || \ defined(PNG_READ_RGB_TO_GRAY_SUPPORTED) - if (png_ptr->transformations & ((PNG_BACKGROUND) | PNG_RGB_TO_GRAY)) + if (png_ptr->transformations & (PNG_COMPOSE | PNG_RGB_TO_GRAY)) { png_build_8bit_table(png_ptr, &png_ptr->gamma_to_1, png_reciprocal(png_ptr->gamma)); @@ -2268,7 +2322,7 @@ png_build_gamma_table(png_structp png_ptr, int bit_depth) png_ptr->screen_gamma > 0 ? png_reciprocal(png_ptr->screen_gamma) : png_ptr->gamma/* Probably doing rgb_to_gray */); } -#endif /* PNG_READ_BACKGROUND_SUPPORTED || PNG_RGB_TO_GRAY_SUPPORTED */ +#endif /* READ_BACKGROUND || READ_ALPHA_MODE || RGB_TO_GRAY */ } else { @@ -2287,7 +2341,7 @@ png_build_gamma_table(png_structp png_ptr, int bit_depth) else sig_bit = png_ptr->sig_bit.gray; - /* 16 bit gamma code uses this equation: + /* 16-bit gamma code uses this equation: * * ov = table[(iv & 0xff) >> gamma_shift][iv >> 8] * @@ -2302,7 +2356,7 @@ png_build_gamma_table(png_structp png_ptr, int bit_depth) * * So the table 'n' corresponds to all those 'iv' of: * - * ..<(n+1 << gamma_shift)-1> + * ..<(n+1 << gamma_shift)-1> * */ if (sig_bit > 0 && sig_bit < 16U) @@ -2311,7 +2365,7 @@ png_build_gamma_table(png_structp png_ptr, int bit_depth) else shift = 0; /* keep all 16 bits */ - if (png_ptr->transformations & PNG_16_TO_8) + if (png_ptr->transformations & (PNG_16_TO_8 | PNG_SCALE_16_TO_8)) { /* PNG_MAX_GAMMA_8 is the number of bits to keep - effectively * the significant bits in the *input* when the output will @@ -2327,7 +2381,12 @@ png_build_gamma_table(png_structp png_ptr, int bit_depth) png_ptr->gamma_shift = shift; #ifdef PNG_16BIT_SUPPORTED - if (png_ptr->transformations & (PNG_16_TO_8 | PNG_BACKGROUND)) + /* NOTE: prior to 1.5.4 this test used to include PNG_BACKGROUND (now + * PNG_COMPOSE). This effectively smashed the background calculation for + * 16-bit output because the 8-bit table assumes the result will be reduced + * to 8 bits. + */ + if (png_ptr->transformations & (PNG_16_TO_8 | PNG_SCALE_16_TO_8)) #endif png_build_16to8_table(png_ptr, &png_ptr->gamma_16_table, shift, png_ptr->screen_gamma > 0 ? png_product2(png_ptr->gamma, @@ -2341,8 +2400,9 @@ png_build_gamma_table(png_structp png_ptr, int bit_depth) #endif #if defined(PNG_READ_BACKGROUND_SUPPORTED) || \ + defined(PNG_READ_ALPHA_MODE_SUPPORTED) || \ defined(PNG_READ_RGB_TO_GRAY_SUPPORTED) - if (png_ptr->transformations & (PNG_BACKGROUND | PNG_RGB_TO_GRAY)) + if (png_ptr->transformations & (PNG_COMPOSE | PNG_RGB_TO_GRAY)) { png_build_16bit_table(png_ptr, &png_ptr->gamma_16_to_1, shift, png_reciprocal(png_ptr->gamma)); @@ -2355,7 +2415,7 @@ png_build_gamma_table(png_structp png_ptr, int bit_depth) png_ptr->screen_gamma > 0 ? png_reciprocal(png_ptr->screen_gamma) : png_ptr->gamma/* Probably doing rgb_to_gray */); } -#endif /* PNG_READ_BACKGROUND_SUPPORTED || PNG_RGB_TO_GRAY_SUPPORTED */ +#endif /* READ_BACKGROUND || READ_ALPHA_MODE || RGB_TO_GRAY */ } } #endif /* READ_GAMMA */ diff --git a/harbour/src/3rd/png/png.dif b/harbour/src/3rd/png/png.dif index 637af9953a..6a38191f65 100644 --- a/harbour/src/3rd/png/png.dif +++ b/harbour/src/3rd/png/png.dif @@ -1,7 +1,7 @@ diff -urN png.orig/png.h png/png.h ---- png.orig/png.h 2011-04-09 09:45:32.341938538 +0200 -+++ png/png.h 2011-04-09 09:45:32.341938538 +0200 -@@ -404,7 +404,7 @@ +--- png.orig/png.h 2011-07-27 17:35:34.724446118 +0200 ++++ png/png.h 2011-07-27 17:35:35.224445268 +0200 +@@ -410,7 +410,7 @@ /* If pnglibconf.h is missing, you can * copy scripts/pnglibconf.h.prebuilt to pnglibconf.h */ @@ -11,8 +11,8 @@ diff -urN png.orig/png.h png/png.h #ifndef PNG_VERSION_INFO_ONLY diff -urN png.orig/pnglconf.h png/pnglconf.h ---- png.orig/pnglconf.h 2011-04-09 09:45:32.251938268 +0200 -+++ png/pnglconf.h 2011-04-09 09:45:32.251938268 +0200 +--- png.orig/pnglconf.h 2011-07-27 17:35:34.604444767 +0200 ++++ png/pnglconf.h 2011-07-27 17:35:35.224445268 +0200 @@ -21,7 +21,11 @@ #ifndef PNGLCONF_H #define PNGLCONF_H diff --git a/harbour/src/3rd/png/png.h b/harbour/src/3rd/png/png.h index 0873b9fa5e..bddb0e48f9 100644 --- a/harbour/src/3rd/png/png.h +++ b/harbour/src/3rd/png/png.h @@ -1,7 +1,7 @@ /* png.h - header file for PNG reference library * - * libpng version 1.5.2 - March 31, 2011 + * libpng version 1.5.4 - July 7, 2011 * Copyright (c) 1998-2011 Glenn Randers-Pehrson * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) @@ -11,7 +11,7 @@ * Authors and maintainers: * libpng versions 0.71, May 1995, through 0.88, January 1996: Guy Schalnat * libpng versions 0.89c, June 1996, through 0.96, May 1997: Andreas Dilger - * libpng versions 0.97, January 1998, through 1.5.2 - March 31, 2011: Glenn + * libpng versions 0.97, January 1998, through 1.5.4 - July 7, 2011: Glenn * See also "Contributing Authors", below. * * Note about libpng version numbers: @@ -150,6 +150,13 @@ * 1.5.2beta01-03 15 10502 15.so.15.2[.0] * 1.5.2rc01-03 15 10502 15.so.15.2[.0] * 1.5.2 15 10502 15.so.15.2[.0] + * 1.5.3beta01-10 15 10503 15.so.15.3[.0] + * 1.5.3rc01-02 15 10503 15.so.15.3[.0] + * 1.5.3beta11 15 10503 15.so.15.3[.0] + * 1.5.3 [omitted] + * 1.5.4beta01-08 15 10504 15.so.15.4[.0] + * 1.5.4rc01 15 10504 15.so.15.4[.0] + * 1.5.4 15 10504 15.so.15.4[.0] * * Henceforth the source version will match the shared-library major * and minor numbers; the shared-library major version number will be @@ -181,7 +188,7 @@ * * This code is released under the libpng license. * - * libpng versions 1.2.6, August 15, 2004, through 1.5.2, March 31, 2011, are + * libpng versions 1.2.6, August 15, 2004, through 1.5.4, July 7, 2011, are * Copyright (c) 2004, 2006-2011 Glenn Randers-Pehrson, and are * distributed according to the same disclaimer and license as libpng-1.2.5 * with the following individual added to the list of Contributing Authors: @@ -293,25 +300,24 @@ * Y2K compliance in libpng: * ========================= * - * March 31, 2011 + * July 7, 2011 * * Since the PNG Development group is an ad-hoc body, we can't make * an official declaration. * * This is your unofficial assurance that libpng from version 0.71 and - * upward through 1.5.2 are Y2K compliant. It is my belief that + * upward through 1.5.4 are Y2K compliant. It is my belief that * earlier versions were also Y2K compliant. * - * Libpng only has three year fields. One is a 2-byte unsigned integer - * that will hold years up to 65535. The other two hold the date in text + * Libpng only has two year fields. One is a 2-byte unsigned integer + * that will hold years up to 65535. The other holds the date in text * format, and will hold years up to 9999. * * The integer is * "png_uint_16 year" in png_time_struct. * - * The strings are - * "png_charp time_buffer" in png_struct and - * "near_time_buffer", which is a local character string in png.c. + * The string is + * "png_char time_buffer" in png_struct * * There are seven time-related functions: * png.c: png_convert_to_rfc_1123() in png.c @@ -355,9 +361,9 @@ */ /* Version information for png.h - this should match the version in png.c */ -#define PNG_LIBPNG_VER_STRING "1.5.2" +#define PNG_LIBPNG_VER_STRING "1.5.4" #define PNG_HEADER_VERSION_STRING \ - " libpng version 1.5.2 - March 31, 2011\n" + " libpng version 1.5.4 - July 7, 2011\n" #define PNG_LIBPNG_VER_SONUM 15 #define PNG_LIBPNG_VER_DLLNUM 15 @@ -365,7 +371,7 @@ /* These should match the first 3 components of PNG_LIBPNG_VER_STRING: */ #define PNG_LIBPNG_VER_MAJOR 1 #define PNG_LIBPNG_VER_MINOR 5 -#define PNG_LIBPNG_VER_RELEASE 2 +#define PNG_LIBPNG_VER_RELEASE 4 /* This should match the numeric part of the final component of * PNG_LIBPNG_VER_STRING, omitting any leading zero: */ @@ -395,7 +401,7 @@ * version 1.0.0 was mis-numbered 100 instead of 10000). From * version 1.0.1 it's xxyyzz, where x=major, y=minor, z=release */ -#define PNG_LIBPNG_VER 10502 /* 1.5.2 */ +#define PNG_LIBPNG_VER 10504 /* 1.5.4 */ /* Library configuration: these options cannot be changed after * the library has been built. @@ -517,7 +523,7 @@ extern "C" { /* This triggers a compiler error in png.c, if png.c and png.h * do not agree upon the version number. */ -typedef char* png_libpng_version_1_5_2; +typedef char* png_libpng_version_1_5_4; /* Three color definitions. The order of the red, green, and blue, (and the * exact size) is not important, although the size of the fields need to @@ -594,7 +600,7 @@ typedef png_sPLT_t FAR * FAR * png_sPLT_tpp; * and whether that contents is compressed or not. The "key" field * points to a regular zero-terminated C string. The "text", "lang", and * "lang_key" fields can be regular C strings, empty strings, or NULL pointers. - * However, the * structure returned by png_get_text() will always contain + * However, the structure returned by png_get_text() will always contain * regular zero-terminated C strings (possibly empty), never NULL pointers, * so they can be safely used in printf() and other string-handling functions. */ @@ -665,12 +671,24 @@ typedef struct png_unknown_chunk_t /* libpng-using applications should NOT directly modify this byte. */ png_byte location; /* mode of operation at read time */ } + + png_unknown_chunk; typedef png_unknown_chunk FAR * png_unknown_chunkp; typedef PNG_CONST png_unknown_chunk FAR * png_const_unknown_chunkp; typedef png_unknown_chunk FAR * FAR * png_unknown_chunkpp; #endif +/* Values for the unknown chunk location byte */ + +#define PNG_HAVE_IHDR 0x01 +#define PNG_HAVE_PLTE 0x02 +#define PNG_AFTER_IDAT 0x08 + +/* The complete definition of png_info has, as of libpng-1.5.0, + * been moved into a separate header file that is not accessible to + * applications. Read libpng-manual.txt or libpng.3 for more info. + */ typedef struct png_info_def png_info; typedef png_info FAR * png_infop; typedef PNG_CONST png_info FAR * png_const_infop; @@ -686,6 +704,8 @@ typedef png_info FAR * FAR * png_infopp; */ #define PNG_FP_1 100000 #define PNG_FP_HALF 50000 +#define PNG_FP_MAX ((png_fixed_point)0x7fffffffL) +#define PNG_FP_MIN (-PNG_FP_MAX) /* These describe the color_type field in png_info. */ /* color type masks */ @@ -792,6 +812,14 @@ typedef struct png_row_info_struct typedef png_row_info FAR * png_row_infop; typedef png_row_info FAR * FAR * png_row_infopp; +/* The complete definition of png_struct has, as of libpng-1.5.0, + * been moved into a separate header file that is not accessible to + * applications. Read libpng-manual.txt or libpng.3 for more info. + */ +typedef struct png_struct_def png_struct; +typedef PNG_CONST png_struct FAR * png_const_structp; +typedef png_struct FAR * png_structp; + /* These are the function types for the I/O functions and for the functions * that allow the user to override the default I/O functions with his or her * own. The png_error_ptr type should match that of user-supplied warning @@ -800,10 +828,6 @@ typedef png_row_info FAR * FAR * png_row_infopp; * modify the buffer it is passed. The 'read' function, on the other hand, is * expected to return the read data in the buffer. */ -typedef struct png_struct_def png_struct; -typedef PNG_CONST png_struct FAR * png_const_structp; -typedef png_struct FAR * png_structp; - typedef PNG_CALLBACK(void, *png_error_ptr, (png_structp, png_const_charp)); typedef PNG_CALLBACK(void, *png_rw_ptr, (png_structp, png_bytep, png_size_t)); typedef PNG_CALLBACK(void, *png_flush_ptr, (png_structp)); @@ -845,25 +869,18 @@ typedef PNG_CALLBACK(void, *png_unknown_chunk_ptr, (png_structp)); #endif #ifdef PNG_SETJMP_SUPPORTED -/* This must match the function definition in , and the - * application must include this before png.h to obtain the definition - * of jmp_buf. The function is required to be PNG_NORETURN. (Note that - * PNG_PTR_NORETURN is used here because current versions of the Microsoft - * C compiler do not support the PNG_NORETURN attribute on a pointer.) +/* This must match the function definition in , and the application + * must include this before png.h to obtain the definition of jmp_buf. The + * function is required to be PNG_NORETURN, but this is not checked. If the + * function does return the application will crash via an abort() or similar + * system level call. * - * If you get a type warning from the compiler when linking against this line - * then your compiler has 'longjmp' that does not match the requirements of the - * compiler that built libpng. You will have to write a wrapper function for - * your compiler's longjmp and call png_set_longjmp_fn directly (not via the - * png_jmpbuf macro.) - * - * If you get a warning here while building the library you will need to make + * If you get a warning here while building the library you may need to make * changes to ensure that pnglibconf.h records the calling convention used by * your compiler. This may be very difficult - try using a different compiler * to build the library! */ -typedef PNG_FUNCTION(void, (PNGCAPI *png_longjmp_ptr), PNGARG((jmp_buf, int)), - PNG_PTR_NORETURN); +PNG_FUNCTION(void, (PNGCAPI *png_longjmp_ptr), PNGARG((jmp_buf, int)), typedef); #endif /* Transform masks for the high-level interface */ @@ -885,6 +902,9 @@ typedef PNG_FUNCTION(void, (PNGCAPI *png_longjmp_ptr), PNGARG((jmp_buf, int)), #define PNG_TRANSFORM_STRIP_FILLER_AFTER 0x1000 /* write only */ /* Added to libpng-1.4.0 */ #define PNG_TRANSFORM_GRAY_TO_RGB 0x2000 /* read only */ +/* Added to libpng-1.5.4 */ +#define PNG_TRANSFORM_EXPAND_16 0x4000 /* read only */ +#define PNG_TRANSFORM_SCALE_16 0x8000 /* read only */ /* Flags for MNG supported features */ #define PNG_FLAG_MNG_EMPTY_PLTE 0x01 @@ -1079,7 +1099,7 @@ PNG_EXPORT(29, void, png_set_tRNS_to_alpha, (png_structp png_ptr)); #endif #ifdef PNG_READ_EXPAND_16_SUPPORTED -/* Expand to 16 bit channels, forces conversion of palette to RGB and expansion +/* Expand to 16-bit channels, forces conversion of palette to RGB and expansion * of a tRNS chunk if present. */ PNG_EXPORT(221, void, png_set_expand_16, (png_structp png_ptr)); @@ -1106,8 +1126,219 @@ PNG_EXPORT(34, png_byte, png_get_rgb_to_gray_status, (png_const_structp png_ptr)); #endif +#ifdef PNG_BUILD_GRAYSCALE_PALETTE_SUPPORTED PNG_EXPORT(35, void, png_build_grayscale_palette, (int bit_depth, png_colorp palette)); +#endif + +#ifdef PNG_READ_ALPHA_MODE_SUPPORTED +/* How the alpha channel is interpreted - this affects how the color channels of + * a PNG file are returned when an alpha channel, or tRNS chunk in a palette + * file, is present. + * + * This has no effect on the way pixels are written into a PNG output + * datastream. The color samples in a PNG datastream are never premultiplied + * with the alpha samples. + * + * The default is to return data according to the PNG specification: the alpha + * channel is a linear measure of the contribution of the pixel to the + * corresponding composited pixel. The gamma encoded color channels must be + * scaled according to the contribution and to do this it is necessary to undo + * the encoding, scale the color values, perform the composition and reencode + * the values. This is the 'PNG' mode. + * + * The alternative is to 'associate' the alpha with the color information by + * storing color channel values that have been scaled by the alpha. The + * advantage is that the color channels can be resampled (the image can be + * scaled) in this form. The disadvantage is that normal practice is to store + * linear, not (gamma) encoded, values and this requires 16-bit channels for + * still images rather than the 8-bit channels that are just about sufficient if + * gamma encoding is used. In addition all non-transparent pixel values, + * including completely opaque ones, must be gamma encoded to produce the final + * image. This is the 'STANDARD', 'ASSOCIATED' or 'PREMULTIPLIED' mode (the + * latter being the two common names for associated alpha color channels.) + * + * Since it is not necessary to perform arithmetic on opaque color values so + * long as they are not to be resampled and are in the final color space it is + * possible to optimize the handling of alpha by storing the opaque pixels in + * the PNG format (adjusted for the output color space) while storing partially + * opaque pixels in the standard, linear, format. The accuracy required for + * standard alpha composition is relatively low, because the pixels are + * isolated, therefore typically the accuracy loss in storing 8-bit linear + * values is acceptable. (This is not true if the alpha channel is used to + * simulate transparency over large areas - use 16 bits or the PNG mode in + * this case!) This is the 'OPTIMIZED' mode. For this mode a pixel is + * treated as opaque only if the alpha value is equal to the maximum value. + * + * The final choice is to gamma encode the alpha channel as well. This is + * broken because, in practice, no implementation that uses this choice + * correctly undoes the encoding before handling alpha composition. Use this + * choice only if other serious errors in the software or hardware you use + * mandate it; the typical serious error is for dark halos to appear around + * opaque areas of the composited PNG image because of arithmetic overflow. + * + * The API function png_set_alpha_mode specifies which of these choices to use + * with an enumerated 'mode' value and the gamma of the required output: + */ +#define PNG_ALPHA_PNG 0 /* according to the PNG standard */ +#define PNG_ALPHA_STANDARD 1 /* according to Porter/Duff */ +#define PNG_ALPHA_ASSOCIATED 1 /* as above; this is the normal practice */ +#define PNG_ALPHA_PREMULTIPLIED 1 /* as above */ +#define PNG_ALPHA_OPTIMIZED 2 /* 'PNG' for opaque pixels, else 'STANDARD' */ +#define PNG_ALPHA_BROKEN 3 /* the alpha channel is gamma encoded */ + +PNG_FP_EXPORT(227, void, png_set_alpha_mode, (png_structp png_ptr, int mode, + double output_gamma)); +PNG_FIXED_EXPORT(228, void, png_set_alpha_mode_fixed, (png_structp png_ptr, + int mode, png_fixed_point output_gamma)); +#endif + +#if defined(PNG_READ_GAMMA_SUPPORTED) || defined(PNG_READ_ALPHA_MODE_SUPPORTED) +/* The output_gamma value is a screen gamma in libpng terminology: it expresses + * how to decode the output values, not how they are encoded. The values used + * correspond to the normal numbers used to describe the overall gamma of a + * computer display system; for example 2.2 for an sRGB conformant system. The + * values are scaled by 100000 in the _fixed version of the API (so 220000 for + * sRGB.) + * + * The inverse of the value is always used to provide a default for the PNG file + * encoding if it has no gAMA chunk and if png_set_gamma() has not been called + * to override the PNG gamma information. + * + * When the ALPHA_OPTIMIZED mode is selected the output gamma is used to encode + * opaque pixels however pixels with lower alpha values are not encoded, + * regardless of the output gamma setting. + * + * When the standard Porter Duff handling is requested with mode 1 the output + * encoding is set to be linear and the output_gamma value is only relevant + * as a default for input data that has no gamma information. The linear output + * encoding will be overridden if png_set_gamma() is called - the results may be + * highly unexpected! + * + * The following numbers are derived from the sRGB standard and the research + * behind it. sRGB is defined to be approximated by a PNG gAMA chunk value of + * 0.45455 (1/2.2) for PNG. The value implicitly includes any viewing + * correction required to take account of any differences in the color + * environment of the original scene and the intended display environment; the + * value expresses how to *decode* the image for display, not how the original + * data was *encoded*. + * + * sRGB provides a peg for the PNG standard by defining a viewing environment. + * sRGB itself, and earlier TV standards, actually use a more complex transform + * (a linear portion then a gamma 2.4 power law) than PNG can express. (PNG is + * limited to simple power laws.) By saying that an image for direct display on + * an sRGB conformant system should be stored with a gAMA chunk value of 45455 + * (11.3.3.2 and 11.3.3.5 of the ISO PNG specification) the PNG specification + * makes it possible to derive values for other display systems and + * environments. + * + * The Mac value is deduced from the sRGB based on an assumption that the actual + * extra viewing correction used in early Mac display systems was implemented as + * a power 1.45 lookup table. + * + * Any system where a programmable lookup table is used or where the behavior of + * the final display device characteristics can be changed requires system + * specific code to obtain the current characteristic. However this can be + * difficult and most PNG gamma correction only requires an approximate value. + * + * By default, if png_set_alpha_mode() is not called, libpng assumes that all + * values are unencoded, linear, values and that the output device also has a + * linear characteristic. This is only very rarely correct - it is invariably + * better to call png_set_alpha_mode() with PNG_DEFAULT_sRGB than rely on the + * default if you don't know what the right answer is! + * + * The special value PNG_GAMMA_MAC_18 indicates an older Mac system (pre Mac OS + * 10.6) which used a correction table to implement a somewhat lower gamma on an + * otherwise sRGB system. + * + * Both these values are reserved (not simple gamma values) in order to allow + * more precise correction internally in the future. + * + * NOTE: the following values can be passed to either the fixed or floating + * point APIs, but the floating point API will also accept floating point + * values. + */ +#define PNG_DEFAULT_sRGB -1 /* sRGB gamma and color space */ +#define PNG_GAMMA_MAC_18 -2 /* Old Mac '1.8' gamma and color space */ +#define PNG_GAMMA_sRGB 220000 /* Television standards--matches sRGB gamma */ +#define PNG_GAMMA_LINEAR PNG_FP_1 /* Linear */ +#endif + +/* The following are examples of calls to png_set_alpha_mode to achieve the + * required overall gamma correction and, where necessary, alpha + * premultiplication. + * + * png_set_alpha_mode(pp, PNG_ALPHA_PNG, PNG_DEFAULT_sRGB); + * This is the default libpng handling of the alpha channel - it is not + * pre-multiplied into the color components. In addition the call states + * that the output is for a sRGB system and causes all PNG files without gAMA + * chunks to be assumed to be encoded using sRGB. + * + * png_set_alpha_mode(pp, PNG_ALPHA_PNG, PNG_GAMMA_MAC); + * In this case the output is assumed to be something like an sRGB conformant + * display preceeded by a power-law lookup table of power 1.45. This is how + * early Mac systems behaved. + * + * png_set_alpha_mode(pp, PNG_ALPHA_STANDARD, PNG_GAMMA_LINEAR); + * This is the classic Jim Blinn approach and will work in academic + * environments where everything is done by the book. It has the shortcoming + * of assuming that input PNG data with no gamma information is linear - this + * is unlikely to be correct unless the PNG files where generated locally. + * Most of the time the output precision will be so low as to show + * significant banding in dark areas of the image. + * + * png_set_expand_16(pp); + * png_set_alpha_mode(pp, PNG_ALPHA_STANDARD, PNG_DEFAULT_sRGB); + * This is a somewhat more realistic Jim Blinn inspired approach. PNG files + * are assumed to have the sRGB encoding if not marked with a gamma value and + * the output is always 16 bits per component. This permits accurate scaling + * and processing of the data. If you know that your input PNG files were + * generated locally you might need to replace PNG_DEFAULT_sRGB with the + * correct value for your system. + * + * png_set_alpha_mode(pp, PNG_ALPHA_OPTIMIZED, PNG_DEFAULT_sRGB); + * If you just need to composite the PNG image onto an existing background + * and if you control the code that does this you can use the optimization + * setting. In this case you just copy completely opaque pixels to the + * output. For pixels that are not completely transparent (you just skip + * those) you do the composition math using png_composite or png_composite_16 + * below then encode the resultant 8-bit or 16-bit values to match the output + * encoding. + * + * Other cases + * If neither the PNG nor the standard linear encoding work for you because + * of the software or hardware you use then you have a big problem. The PNG + * case will probably result in halos around the image. The linear encoding + * will probably result in a washed out, too bright, image (it's actually too + * contrasty.) Try the ALPHA_OPTIMIZED mode above - this will probably + * substantially reduce the halos. Alternatively try: + * + * png_set_alpha_mode(pp, PNG_ALPHA_BROKEN, PNG_DEFAULT_sRGB); + * This option will also reduce the halos, but there will be slight dark + * halos round the opaque parts of the image where the background is light. + * In the OPTIMIZED mode the halos will be light halos where the background + * is dark. Take your pick - the halos are unavoidable unless you can get + * your hardware/software fixed! (The OPTIMIZED approach is slightly + * faster.) + * + * When the default gamma of PNG files doesn't match the output gamma. + * If you have PNG files with no gamma information png_set_alpha_mode allows + * you to provide a default gamma, but it also sets the ouput gamma to the + * matching value. If you know your PNG files have a gamma that doesn't + * match the output you can take advantage of the fact that + * png_set_alpha_mode always sets the output gamma but only sets the PNG + * default if it is not already set: + * + * png_set_alpha_mode(pp, PNG_ALPHA_PNG, PNG_DEFAULT_sRGB); + * png_set_alpha_mode(pp, PNG_ALPHA_PNG, PNG_GAMMA_MAC); + * The first call sets both the default and the output gamma values, the + * second call overrides the output gamma without changing the default. This + * is easier than achieving the same effect with png_set_gamma. You must use + * PNG_ALPHA_PNG for the first call - internal checking in png_set_alpha will + * fire if more than one call to png_set_alpha_mode and png_set_background is + * made in the same read operation, however multiple calls with PNG_ALPHA_PNG + * are ignored. + */ #ifdef PNG_READ_STRIP_ALPHA_SUPPORTED PNG_EXPORT(36, void, png_set_strip_alpha, (png_structp png_ptr)); @@ -1175,7 +1406,11 @@ PNG_EXPORT(46, void, png_set_invert_mono, (png_structp png_ptr)); #endif #ifdef PNG_READ_BACKGROUND_SUPPORTED -/* Handle alpha and tRNS by replacing with a background color. */ +/* Handle alpha and tRNS by replacing with a background color. Prior to + * libpng-1.5.4 this API must not be called before the PNG file header has been + * read. Doing so will result in unexpected behavior and possible warnings or + * errors if the PNG file contains a bKGD chunk. + */ PNG_FP_EXPORT(47, void, png_set_background, (png_structp png_ptr, png_const_color_16p background_color, int background_gamma_code, int need_expand, double background_gamma)); @@ -1190,7 +1425,13 @@ PNG_FIXED_EXPORT(215, void, png_set_background_fixed, (png_structp png_ptr, # define PNG_BACKGROUND_GAMMA_UNIQUE 3 #endif -#ifdef PNG_READ_16_TO_8_SUPPORTED +#ifdef PNG_READ_SCALE_16_TO_8_SUPPORTED +/* Scale a 16-bit depth file down to 8-bit, accurately. */ +PNG_EXPORT(229, void, png_set_scale_16, (png_structp png_ptr)); +#endif + +#ifdef PNG_READ_STRIP_16_TO_8_SUPPORTED +#define PNG_READ_16_TO_8 SUPPORTED /* Name prior to 1.5.4 */ /* Strip the second byte of information from a 16-bit depth file. */ PNG_EXPORT(48, void, png_set_strip_16, (png_structp png_ptr)); #endif @@ -1211,12 +1452,22 @@ PNG_EXPORT(49, void, png_set_quantize, */ #define PNG_GAMMA_THRESHOLD (PNG_GAMMA_THRESHOLD_FIXED*.00001) -/* Handle gamma correction. Screen_gamma=(display_exponent) */ +/* Handle gamma correction. Screen_gamma=(display_exponent). + * NOTE: this API simply sets the screen and file gamma values. It will + * therefore override the value for gamma in a PNG file if it is called after + * the file header has been read - use with care - call before reading the PNG + * file for best results! + * + * These routines accept the same gamma values as png_set_alpha_mode (described + * above). The PNG_GAMMA_ defines and PNG_DEFAULT_sRGB can be passed to either + * API (floating point or fixed.) Notice, however, that the 'file_gamma' value + * is the inverse of a 'screen gamma' value. + */ PNG_FP_EXPORT(50, void, png_set_gamma, (png_structp png_ptr, double screen_gamma, - double default_file_gamma)); + double override_file_gamma)); PNG_FIXED_EXPORT(208, void, png_set_gamma_fixed, (png_structp png_ptr, - png_fixed_point screen_gamma, png_fixed_point default_file_gamma)); + png_fixed_point screen_gamma, png_fixed_point override_file_gamma)); #endif #ifdef PNG_WRITE_FLUSH_SUPPORTED @@ -1391,6 +1642,7 @@ PNG_FIXED_EXPORT(209, void, png_set_filter_heuristics_fixed, #define PNG_FILTER_HEURISTIC_WEIGHTED 2 /* Experimental feature */ #define PNG_FILTER_HEURISTIC_LAST 3 /* Not a valid value */ +#ifdef PNG_WRITE_SUPPORTED /* Set the library compression level. Currently, valid values range from * 0 - 9, corresponding directly to the zlib compression levels 0 - 9 * (0 - no compression, 9 - "maximal" compression). Note that tests have @@ -1407,11 +1659,36 @@ PNG_EXPORT(70, void, png_set_compression_mem_level, (png_structp png_ptr, PNG_EXPORT(71, void, png_set_compression_strategy, (png_structp png_ptr, int strategy)); +/* If PNG_WRITE_OPTIMIZE_CMF_SUPPORTED is defined, libpng will use a + * smaller value of window_bits if it can do so safely. + */ PNG_EXPORT(72, void, png_set_compression_window_bits, (png_structp png_ptr, int window_bits)); PNG_EXPORT(73, void, png_set_compression_method, (png_structp png_ptr, int method)); +#endif + +#ifdef PNG_WRITE_CUSTOMIZE_ZTXT_COMPRESSION_SUPPORTED +/* Also set zlib parameters for compressing non-IDAT chunks */ +PNG_EXPORT(222, void, png_set_text_compression_level, + (png_structp png_ptr, int level)); + +PNG_EXPORT(223, void, png_set_text_compression_mem_level, (png_structp png_ptr, + int mem_level)); + +PNG_EXPORT(224, void, png_set_text_compression_strategy, (png_structp png_ptr, + int strategy)); + +/* If PNG_WRITE_OPTIMIZE_CMF_SUPPORTED is defined, libpng will use a + * smaller value of window_bits if it can do so safely. + */ +PNG_EXPORT(225, void, png_set_text_compression_window_bits, (png_structp + png_ptr, int window_bits)); + +PNG_EXPORT(226, void, png_set_text_compression_method, (png_structp png_ptr, + int method)); +#endif /* PNG_WRITE_CUSTOMIZE_ZTXT_COMPRESSION_SUPPORTED */ /* These next functions are called for input/output, memory, and error * handling. They are in the file pngrio.c, pngwio.c, and pngerror.c, @@ -1623,6 +1900,7 @@ PNG_EXPORTA(103, void, png_chunk_error, (png_structp png_ptr, PNG_EXPORTA(104, void, png_err, (png_structp png_ptr), PNG_NORETURN); #endif +#ifdef PNG_WARNINGS_SUPPORTED /* Non-fatal error in libpng. Can continue, but may have a problem. */ PNG_EXPORT(105, void, png_warning, (png_structp png_ptr, png_const_charp warning_message)); @@ -1630,6 +1908,7 @@ PNG_EXPORT(105, void, png_warning, (png_structp png_ptr, /* Non-fatal error in libpng, chunk name is prepended to message. */ PNG_EXPORT(106, void, png_chunk_warning, (png_structp png_ptr, png_const_charp warning_message)); +#endif #ifdef PNG_BENIGN_ERRORS_SUPPORTED /* Benign error in libpng. Can continue, but may have a problem. @@ -2297,7 +2576,7 @@ PNG_EXPORT(207, void, png_save_uint_16, (png_bytep buf, unsigned int i)); * scripts/symbols.def as well. */ #ifdef PNG_EXPORT_LAST_ORDINAL - PNG_EXPORT_LAST_ORDINAL(221); + PNG_EXPORT_LAST_ORDINAL(229); #endif #ifdef __cplusplus diff --git a/harbour/src/3rd/png/pngconf.h b/harbour/src/3rd/png/pngconf.h index c82fa5866c..222816e372 100644 --- a/harbour/src/3rd/png/pngconf.h +++ b/harbour/src/3rd/png/pngconf.h @@ -1,7 +1,7 @@ /* pngconf.h - machine configurable file for libpng * - * libpng version 1.5.2 - March 31, 2011 + * libpng version 1.5.4 - July 7, 2011 * * Copyright (c) 1998-2011 Glenn Randers-Pehrson * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) @@ -353,23 +353,6 @@ # ifndef PNG_NORETURN # define PNG_NORETURN __attribute__((__noreturn__)) # endif -# ifndef PNG_PTR_NORETURN - /* It's not enough to have the compiler be the correct compiler at - * this point - it's necessary for the library (which defines - * the type of the library longjmp) to also be the GNU library. - * This is because many systems use the GNU compiler with a - * non-GNU libc implementation. Min/GW headers are also compatible - * with GCC as well as uclibc, so it seems best to exclude known - * problem libcs here rather than just including known libcs. - * - * NOTE: this relies on the only use of PNG_PTR_NORETURN being with - * the system longjmp. If the same type is used elsewhere then this - * will need to be changed. - */ -# if !defined(__CYGWIN__) -# define PNG_PTR_NORETURN __attribute__((__noreturn__)) -# endif -# endif # ifndef PNG_ALLOCATED # define PNG_ALLOCATED __attribute__((__malloc__)) # endif @@ -382,9 +365,6 @@ # ifndef PNG_DEPRECATED # define PNG_DEPRECATED __attribute__((__deprecated__)) # endif -# ifndef PNG_DEPSTRUCT -# define PNG_DEPSTRUCT __attribute__((__deprecated__)) -# endif # ifndef PNG_PRIVATE # if 0 /* Doesn't work so we use deprecated instead*/ # define PNG_PRIVATE \ @@ -404,11 +384,10 @@ # ifndef PNG_NORETURN # define PNG_NORETURN __declspec(noreturn) # endif -# ifndef PNG_PTR_NORETURN -# define PNG_PTR_NORETURN /* not supported */ -# endif # ifndef PNG_ALLOCATED -# define PNG_ALLOCATED __declspec(restrict) +# if (_MSC_VER >= 1400) +# define PNG_ALLOCATED __declspec(restrict) +# endif # endif /* This specifically protects structure members that should only be @@ -419,9 +398,6 @@ # ifndef PNG_DEPRECATED # define PNG_DEPRECATED __declspec(deprecated) # endif -# ifndef PNG_DEPSTRUCT -# define PNG_DEPSTRUCT __declspec(deprecated) -# endif # ifndef PNG_PRIVATE # define PNG_PRIVATE __declspec(deprecated) # endif @@ -438,15 +414,9 @@ #ifndef PNG_NORETURN # define PNG_NORETURN /* This function does not return */ #endif -#ifndef PNG_PTR_NORETURN -# define PNG_PTR_NORETURN /* This function does not return */ -#endif #ifndef PNG_ALLOCATED # define PNG_ALLOCATED /* The result of the function is new memory */ #endif -#ifndef PNG_DEPSTRUCT -# define PNG_DEPSTRUCT /* Access to this struct member is deprecated */ -#endif #ifndef PNG_PRIVATE # define PNG_PRIVATE /* This is a private libpng function */ #endif diff --git a/harbour/src/3rd/png/pngerror.c b/harbour/src/3rd/png/pngerror.c index 8290bb4106..4d4cebafa9 100644 --- a/harbour/src/3rd/png/pngerror.c +++ b/harbour/src/3rd/png/pngerror.c @@ -1,7 +1,7 @@ /* pngerror.c - stub functions for i/o and memory allocation * - * Last changed in libpng 1.5.1 [February 3, 2011] + * Last changed in libpng 1.5.4 [July 7, 2011] * Copyright (c) 1998-2011 Glenn Randers-Pehrson * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) @@ -89,15 +89,121 @@ png_error,(png_structp png_ptr, png_const_charp error_message),PNG_NORETURN) PNG_FUNCTION(void,PNGAPI png_err,(png_structp png_ptr),PNG_NORETURN) { + /* Prior to 1.5.2 the error_fn received a NULL pointer, expressed + * erroneously as '\0', instead of the empty string "". This was + * apparently an error, introduced in libpng-1.2.20, and png_default_error + * will crash in this case. + */ if (png_ptr != NULL && png_ptr->error_fn != NULL) - (*(png_ptr->error_fn))(png_ptr, '\0'); + (*(png_ptr->error_fn))(png_ptr, ""); /* If the custom handler doesn't exist, or if it returns, use the default handler, which will not return. */ - png_default_error(png_ptr, '\0'); + png_default_error(png_ptr, ""); } #endif /* PNG_ERROR_TEXT_SUPPORTED */ +/* Utility to safely appends strings to a buffer. This never errors out so + * error checking is not required in the caller. + */ +size_t +png_safecat(png_charp buffer, size_t bufsize, size_t pos, + png_const_charp string) +{ + if (buffer != NULL && pos < bufsize) + { + if (string != NULL) + while (*string != '\0' && pos < bufsize-1) + buffer[pos++] = *string++; + + buffer[pos] = '\0'; + } + + return pos; +} + +#if defined(PNG_WARNINGS_SUPPORTED) || defined(PNG_TIME_RFC1123_SUPPORTED) +/* Utility to dump an unsigned value into a buffer, given a start pointer and + * and end pointer (which should point just *beyond* the end of the buffer!) + * Returns the pointer to the start of the formatted string. + */ +png_charp +png_format_number(png_const_charp start, png_charp end, int format, + png_alloc_size_t number) +{ + int count = 0; /* number of digits output */ + int mincount = 1; /* minimum number required */ + int output = 0; /* digit output (for the fixed point format) */ + + *--end = '\0'; + + /* This is written so that the loop always runs at least once, even with + * number zero. + */ + while (end > start && (number != 0 || count < mincount)) + { + + static const char digits[] = "0123456789ABCDEF"; + + switch (format) + { + case PNG_NUMBER_FORMAT_fixed: + /* Needs five digits (the fraction) */ + mincount = 5; + if (output || number % 10 != 0) + { + *--end = digits[number % 10]; + output = 1; + } + number /= 10; + break; + + case PNG_NUMBER_FORMAT_02u: + /* Expects at least 2 digits. */ + mincount = 2; + /* fall through */ + + case PNG_NUMBER_FORMAT_u: + *--end = digits[number % 10]; + number /= 10; + break; + + case PNG_NUMBER_FORMAT_02x: + /* This format expects at least two digits */ + mincount = 2; + /* fall through */ + + case PNG_NUMBER_FORMAT_x: + *--end = digits[number & 0xf]; + number >>= 4; + break; + + default: /* an error */ + number = 0; + break; + } + + /* Keep track of the number of digits added */ + ++count; + + /* Float a fixed number here: */ + if (format == PNG_NUMBER_FORMAT_fixed) if (count == 5) if (end > start) + { + /* End of the fraction, but maybe nothing was output? In that case + * drop the decimal point. If the number is a true zero handle that + * here. + */ + if (output) + *--end = '.'; + else if (number == 0) /* and !output */ + *--end = '0'; + } + } + + return end; +} +#endif + #ifdef PNG_WARNINGS_SUPPORTED /* This function is called whenever there is a non-fatal error. This function * should not be changed. If there is a need to handle warnings differently, @@ -128,6 +234,115 @@ png_warning(png_structp png_ptr, png_const_charp warning_message) else png_default_warning(png_ptr, warning_message + offset); } + +/* These functions support 'formatted' warning messages with up to + * PNG_WARNING_PARAMETER_COUNT parameters. In the format string the parameter + * is introduced by @, where 'number' starts at 1. This follows the + * standard established by X/Open for internationalizable error messages. + */ +void +png_warning_parameter(png_warning_parameters p, int number, + png_const_charp string) +{ + if (number > 0 && number <= PNG_WARNING_PARAMETER_COUNT) + (void)png_safecat(p[number-1], (sizeof p[number-1]), 0, string); +} + +void +png_warning_parameter_unsigned(png_warning_parameters p, int number, int format, + png_alloc_size_t value) +{ + char buffer[PNG_NUMBER_BUFFER_SIZE]; + png_warning_parameter(p, number, PNG_FORMAT_NUMBER(buffer, format, value)); +} + +void +png_warning_parameter_signed(png_warning_parameters p, int number, int format, + png_int_32 value) +{ + png_alloc_size_t u; + png_charp str; + char buffer[PNG_NUMBER_BUFFER_SIZE]; + + /* Avoid overflow by doing the negate in a png_alloc_size_t: */ + u = (png_alloc_size_t)value; + if (value < 0) + u = ~u + 1; + + str = PNG_FORMAT_NUMBER(buffer, format, u); + + if (value < 0 && str > buffer) + *--str = '-'; + + png_warning_parameter(p, number, str); +} + +void +png_formatted_warning(png_structp png_ptr, png_warning_parameters p, + png_const_charp message) +{ + /* The internal buffer is just 128 bytes - enough for all our messages, + * overflow doesn't happen because this code checks! + */ + size_t i; + char msg[128]; + + for (i=0; i<(sizeof msg)-1 && *message != '\0'; ++i) + { + if (*message == '@') + { + int parameter = -1; + switch (*++message) + { + case '1': + parameter = 0; + break; + + case '2': + parameter = 1; + break; + + case '\0': + continue; /* To break out of the for loop above. */ + + default: + break; + } + + if (parameter >= 0 && parameter < PNG_WARNING_PARAMETER_COUNT) + { + /* Append this parameter */ + png_const_charp parm = p[parameter]; + png_const_charp pend = p[parameter] + (sizeof p[parameter]); + + /* No need to copy the trailing '\0' here, but there is no guarantee + * that parm[] has been initialized, so there is no guarantee of a + * trailing '\0': + */ + for (; i<(sizeof msg)-1 && parm != '\0' && parm < pend; ++i) + msg[i] = *parm++; + + ++message; + continue; + } + + /* else not a parameter and there is a character after the @ sign; just + * copy that. + */ + } + + /* At this point *message can't be '\0', even in the bad parameter case + * above where there is a lone '@' at the end of the message string. + */ + msg[i] = *message++; + } + + /* i is always less than (sizeof msg), so: */ + msg[i] = '\0'; + + /* And this is the formatted message: */ + png_warning(png_ptr, msg); +} #endif /* PNG_WARNINGS_SUPPORTED */ #ifdef PNG_BENIGN_ERRORS_SUPPORTED @@ -185,8 +400,13 @@ png_format_buffer(png_structp png_ptr, png_charp buffer, png_const_charp { buffer[iout++] = ':'; buffer[iout++] = ' '; - png_memcpy(buffer + iout, error_message, PNG_MAX_ERROR_TEXT); - buffer[iout + PNG_MAX_ERROR_TEXT - 1] = '\0'; + + iin = 0; + while (iin < PNG_MAX_ERROR_TEXT-1 && error_message[iin] != '\0') + buffer[iout++] = error_message[iin++]; + + /* iin < PNG_MAX_ERROR_TEXT, so the following is safe: */ + buffer[iout] = '\0'; } } #endif /* PNG_WARNINGS_SUPPORTED || PNG_ERROR_TEXT_SUPPORTED */ @@ -272,7 +492,7 @@ png_set_longjmp_fn(png_structp png_ptr, png_longjmp_ptr longjmp_fn, return NULL; png_ptr->longjmp_fn = longjmp_fn; - return &png_ptr->png_jmpbuf; + return &png_ptr->longjmp_buffer; } #endif @@ -287,7 +507,8 @@ png_default_error,(png_structp png_ptr, png_const_charp error_message), { #ifdef PNG_CONSOLE_IO_SUPPORTED #ifdef PNG_ERROR_NUMBERS_SUPPORTED - if (*error_message == PNG_LITERAL_SHARP) + /* Check on NULL only added in 1.5.4 */ + if (error_message != NULL && *error_message == PNG_LITERAL_SHARP) { /* Strip "#nnnn " from beginning of error message. */ int offset; @@ -317,11 +538,11 @@ png_default_error,(png_structp png_ptr, png_const_charp error_message), else #endif { - fprintf(stderr, "libpng error: %s", error_message); + fprintf(stderr, "libpng error: %s", error_message ? error_message : + "undefined"); fprintf(stderr, PNG_STRING_NEWLINE); } -#endif -#ifndef PNG_CONSOLE_IO_SUPPORTED +#else PNG_UNUSED(error_message) /* Make compiler happy */ #endif png_longjmp(png_ptr, 1); @@ -335,13 +556,13 @@ png_longjmp,(png_structp png_ptr, int val),PNG_NORETURN) { # ifdef USE_FAR_KEYWORD { - jmp_buf png_jmpbuf; - png_memcpy(png_jmpbuf, png_ptr->png_jmpbuf, png_sizeof(jmp_buf)); - png_ptr->longjmp_fn(png_jmpbuf, val); + jmp_buf tmp_jmpbuf; + png_memcpy(tmp_jmpbuf, png_ptr->longjmp_buffer, png_sizeof(jmp_buf)); + png_ptr->longjmp_fn(tmp_jmpbuf, val); } # else - png_ptr->longjmp_fn(png_ptr->png_jmpbuf, val); + png_ptr->longjmp_fn(png_ptr->longjmp_buffer, val); # endif } #endif @@ -403,7 +624,7 @@ png_default_warning(png_structp png_ptr, png_const_charp warning_message) /* This function is called when the application wants to use another method * of handling errors and warnings. Note that the error function MUST NOT * return to the calling routine or serious problems will occur. The return - * method used in the default routine calls longjmp(png_ptr->png_jmpbuf, 1) + * method used in the default routine calls longjmp(png_ptr->longjmp_buffer, 1) */ void PNGAPI png_set_error_fn(png_structp png_ptr, png_voidp error_ptr, @@ -414,7 +635,11 @@ png_set_error_fn(png_structp png_ptr, png_voidp error_ptr, png_ptr->error_ptr = error_ptr; png_ptr->error_fn = error_fn; +#ifdef PNG_WARNINGS_SUPPORTED png_ptr->warning_fn = warning_fn; +#else + PNG_UNUSED(warning_fn) +#endif } diff --git a/harbour/src/3rd/png/pnginfo.h b/harbour/src/3rd/png/pnginfo.h index fa19f85e72..a33bfab06d 100644 --- a/harbour/src/3rd/png/pnginfo.h +++ b/harbour/src/3rd/png/pnginfo.h @@ -138,7 +138,6 @@ defined(PNG_READ_BACKGROUND_SUPPORTED) * single color specified that should be treated as fully transparent. * Data is valid if (valid & PNG_INFO_tRNS) is non-zero. */ - png_bytep trans; /* alpha values for paletted image */ png_bytep trans_alpha; /* alpha values for paletted image */ png_color_16 trans_color; /* transparent color for non-palette image */ #endif diff --git a/harbour/src/3rd/png/pnglconf.h b/harbour/src/3rd/png/pnglconf.h index c80062c574..56dcabce0e 100644 --- a/harbour/src/3rd/png/pnglconf.h +++ b/harbour/src/3rd/png/pnglconf.h @@ -3,7 +3,7 @@ /* pnglibconf.h - library build configuration */ -/* libpng version 1.5.0 - last changed on February 11, 2011 */ +/* libpng version 1.5.4 - last changed on June 22, 2011 */ /* Copyright (c) 1998-2011 Glenn Randers-Pehrson */ @@ -17,7 +17,7 @@ /* pnglibconf.dfa with respect to the dependencies between the following */ /* symbols. It is much better to generate a new file using */ /* scripts/libpngconf.mak */ - + #ifndef PNGLCONF_H #define PNGLCONF_H /* settings */ @@ -47,11 +47,13 @@ #define PNG_ALIGN_MEMORY_SUPPORTED #define PNG_BENIGN_ERRORS_SUPPORTED #define PNG_bKGD_SUPPORTED +#define PNG_BUILD_GRAYSCALE_PALETTE_SUPPORTED #define PNG_CHECK_cHRM_SUPPORTED #define PNG_cHRM_SUPPORTED #define PNG_CONSOLE_IO_SUPPORTED #define PNG_CONVERT_tIME_SUPPORTED #define PNG_EASY_ACCESS_SUPPORTED +/*#undef PNG_ERROR_NUMBERS_SUPPORTED*/ #define PNG_ERROR_TEXT_SUPPORTED #define PNG_FIXED_POINT_SUPPORTED #define PNG_FLOATING_ARITHMETIC_SUPPORTED @@ -71,13 +73,14 @@ #define PNG_POINTER_INDEXING_SUPPORTED #define PNG_PROGRESSIVE_READ_SUPPORTED #define PNG_READ_16BIT_SUPPORTED -#define PNG_READ_16_TO_8_SUPPORTED +#define PNG_READ_ALPHA_MODE_SUPPORTED #define PNG_READ_ANCILLARY_CHUNKS_SUPPORTED #define PNG_READ_BACKGROUND_SUPPORTED #define PNG_READ_BGR_SUPPORTED #define PNG_READ_bKGD_SUPPORTED #define PNG_READ_cHRM_SUPPORTED #define PNG_READ_COMPOSITE_NODIV_SUPPORTED +#define PNG_READ_COMPRESSED_TEXT_SUPPORTED #define PNG_READ_EXPAND_16_SUPPORTED #define PNG_READ_EXPAND_SUPPORTED #define PNG_READ_FILLER_SUPPORTED @@ -100,10 +103,12 @@ #define PNG_READ_QUANTIZE_SUPPORTED #define PNG_READ_RGB_TO_GRAY_SUPPORTED #define PNG_READ_sBIT_SUPPORTED +#define PNG_READ_SCALE_16_TO_8_SUPPORTED #define PNG_READ_sCAL_SUPPORTED #define PNG_READ_SHIFT_SUPPORTED #define PNG_READ_sPLT_SUPPORTED #define PNG_READ_sRGB_SUPPORTED +#define PNG_READ_STRIP_16_TO_8_SUPPORTED #define PNG_READ_STRIP_ALPHA_SUPPORTED #define PNG_READ_SUPPORTED #define PNG_READ_SWAP_ALPHA_SUPPORTED @@ -145,6 +150,8 @@ #define PNG_WRITE_BGR_SUPPORTED #define PNG_WRITE_bKGD_SUPPORTED #define PNG_WRITE_cHRM_SUPPORTED +#define PNG_WRITE_COMPRESSED_TEXT_SUPPORTED +#define PNG_WRITE_CUSTOMIZE_ZTXT_COMPRESSION_SUPPORTED #define PNG_WRITE_FILLER_SUPPORTED #define PNG_WRITE_FILTER_SUPPORTED #define PNG_WRITE_FLUSH_SUPPORTED @@ -157,6 +164,7 @@ #define PNG_WRITE_INVERT_SUPPORTED #define PNG_WRITE_iTXt_SUPPORTED #define PNG_WRITE_oFFs_SUPPORTED +#define PNG_WRITE_OPTIMIZE_CMF_SUPPORTED #define PNG_WRITE_PACK_SUPPORTED #define PNG_WRITE_PACKSWAP_SUPPORTED #define PNG_WRITE_pCAL_SUPPORTED @@ -179,7 +187,5 @@ #define PNG_WRITE_WEIGHTED_FILTER_SUPPORTED #define PNG_WRITE_zTXt_SUPPORTED #define PNG_zTXt_SUPPORTED -/*#undef PNG_ERROR_NUMBERS_SUPPORTED*/ -/*#undef PNG_READ_16_TO_8_ACCURATE_SCALE_SUPPORTED*/ /* end of options */ #endif /* PNGLCONF_H */ diff --git a/harbour/src/3rd/png/pngmem.c b/harbour/src/3rd/png/pngmem.c index a15d8b0859..ea606d6a62 100644 --- a/harbour/src/3rd/png/pngmem.c +++ b/harbour/src/3rd/png/pngmem.c @@ -1,7 +1,7 @@ /* pngmem.c - stub functions for memory allocation * - * Last changed in libpng 1.5.1 [February 3, 2011] + * Last changed in libpng 1.5.4 [July 7, 2011] * Copyright (c) 1998-2011 Glenn Randers-Pehrson * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) @@ -187,8 +187,9 @@ png_malloc_default,(png_structp png_ptr, png_alloc_size_t size),PNG_ALLOCATED) int num_blocks; png_uint_32 total_size; png_bytep table; - int i; + int i, mem_level, window_bits; png_byte huge * hptr; + int window_bits if (ret != NULL) { @@ -196,14 +197,22 @@ png_malloc_default,(png_structp png_ptr, png_alloc_size_t size),PNG_ALLOCATED) ret = NULL; } - if (png_ptr->zlib_window_bits > 14) - num_blocks = (int)(1 << (png_ptr->zlib_window_bits - 14)); + window_bits = + png_ptr->zlib_window_bits >= png_ptr->zlib_text_window_bits ? + png_ptr->zlib_window_bits : png_ptr->zlib_text_window_bits; + + if (window_bits > 14) + num_blocks = (int)(1 << (window_bits - 14)); else num_blocks = 1; - if (png_ptr->zlib_mem_level >= 7) - num_blocks += (int)(1 << (png_ptr->zlib_mem_level - 7)); + mem_level = + png_ptr->zlib_mem_level >= png_ptr->zlib_text_mem_level ? + png_ptr->zlib_mem_level : png_ptr->zlib_text_mem_level; + + if (mem_level >= 7) + num_blocks += (int)(1 << (mem_level - 7)); else num_blocks++; @@ -277,7 +286,7 @@ png_malloc_default,(png_structp png_ptr, png_alloc_size_t size),PNG_ALLOCATED) { # ifndef PNG_USER_MEM_SUPPORTED if ((png_ptr->flags&PNG_FLAG_MALLOC_NULL_MEM_OK) == 0) - png_error(png_ptr, "Out of Memory"); /* Note "o" and "M" */ + png_error(png_ptr, "Out of Memory"); /* Note "O" and "M" */ else png_warning(png_ptr, "Out of Memory"); diff --git a/harbour/src/3rd/png/pngpread.c b/harbour/src/3rd/png/pngpread.c index 5ab9007a16..a50292a76a 100644 --- a/harbour/src/3rd/png/pngpread.c +++ b/harbour/src/3rd/png/pngpread.c @@ -1026,8 +1026,10 @@ png_push_process_row(png_structp png_ptr) png_memcpy(png_ptr->prev_row, png_ptr->row_buf, png_ptr->rowbytes + 1); +#ifdef PNG_READ_TRANSFORMS_SUPPORTED if (png_ptr->transformations) png_do_read_transformations(png_ptr); +#endif #ifdef PNG_READ_INTERLACING_SUPPORTED /* Blow up interlaced rows to full size */ @@ -1288,7 +1290,7 @@ png_push_handle_tEXt(png_structp png_ptr, png_infop info_ptr, png_uint_32 { PNG_UNUSED(info_ptr) /* To quiet some compiler warnings */ png_error(png_ptr, "Out of place tEXt"); - /*NOT REACHED*/ + /* NOT REACHED */ } #ifdef PNG_MAX_MALLOC_64K @@ -1385,7 +1387,7 @@ png_push_handle_zTXt(png_structp png_ptr, png_infop info_ptr, png_uint_32 { PNG_UNUSED(info_ptr) /* To quiet some compiler warnings */ png_error(png_ptr, "Out of place zTXt"); - /*NOT REACHED*/ + /* NOT REACHED */ } #ifdef PNG_MAX_MALLOC_64K @@ -1589,7 +1591,7 @@ png_push_handle_iTXt(png_structp png_ptr, png_infop info_ptr, png_uint_32 { PNG_UNUSED(info_ptr) /* To quiet some compiler warnings */ png_error(png_ptr, "Out of place iTXt"); - /*NOT REACHED*/ + /* NOT REACHED */ } #ifdef PNG_MAX_MALLOC_64K diff --git a/harbour/src/3rd/png/pngpriv.h b/harbour/src/3rd/png/pngpriv.h index b922f420e6..d205b88127 100644 --- a/harbour/src/3rd/png/pngpriv.h +++ b/harbour/src/3rd/png/pngpriv.h @@ -6,7 +6,7 @@ * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) * - * Last changed in libpng 1.5.2 [March 31, 2011] + * Last changed in libpng 1.5.4 [July 7, 2011] * * This code is released under the libpng license. * For conditions of distribution and use, see the disclaimer @@ -25,6 +25,20 @@ #ifndef PNGPRIV_H #define PNGPRIV_H +/* Feature Test Macros. The following are defined here to ensure that correctly + * implemented libraries reveal the APIs libpng needs to build and hide those + * that are not needed and potentially damaging to the compilation. + * + * Feature Test Macros must be defined before any system header is included (see + * POSIX 1003.1 2.8.2 "POSIX Symbols." + * + * These macros only have an effect if the operating system supports either + * POSIX 1003.1 or C99, or both. On other operating systems (particularly + * Windows/Visual Studio) there is no effect; the OS specific tests below are + * still required (as of 2011-05-02.) + */ +#define _POSIX_SOURCE 1 /* Just the POSIX 1003.1 and C89 APIs */ + /* This is required for the definition of abort(), used as a last ditch * error handler when all else fails. */ @@ -101,12 +115,27 @@ typedef PNG_CONST png_uint_16p FAR * png_const_uint_16pp; # define PNG_ZBUF_SIZE 65536L #endif -/* If warnings or errors are turned off the code is disabled - * or redirected here. +/* PNG_STATIC is used to mark internal file scope functions if they need to be + * accessed for implementation tests (see the code in tests/?*). */ -#ifndef PNG_WARNINGS_SUPPORTED -# define png_warning(s1,s2) ((void)0) -# define png_chunk_warning(s1,s2) ((void)0) +#ifndef PNG_STATIC +# define PNG_STATIC static +#endif + +/* If warnings or errors are turned off the code is disabled or redirected here. + * From 1.5.4 functions have been added to allow very limited formatting of + * error and warning messages - this code will also be disabled here. + */ +#ifdef PNG_WARNINGS_SUPPORTED +# define PNG_WARNING_PARAMETERS(p) png_warning_parameters p; +#else +# define png_warning(s1,s2) ((void)(s1)) +# define png_chunk_warning(s1,s2) ((void)(s1)) +# define png_warning_parameter(p,number,string) ((void)0) +# define png_warning_parameter_unsigned(p,number,format,value) ((void)0) +# define png_warning_parameter_signed(p,number,format,value) ((void)0) +# define png_formatted_warning(pp,p,message) ((void)(pp)) +# define PNG_WARNING_PARAMETERS(p) #endif #ifndef PNG_ERROR_TEXT_SUPPORTED # define png_error(s1,s2) png_err(s1) @@ -200,60 +229,28 @@ typedef PNG_CONST png_uint_16p FAR * png_const_uint_16pp; # define NOCHECK 0 # define CVT_PTR(ptr) (png_far_to_near(png_ptr,ptr,CHECK)) # define CVT_PTR_NOCHECK(ptr) (png_far_to_near(png_ptr,ptr,NOCHECK)) -# define png_strcpy _fstrcpy -# define png_strncpy _fstrncpy /* Added to v 1.2.6 */ # define png_strlen _fstrlen # define png_memcmp _fmemcmp /* SJT: added */ # define png_memcpy _fmemcpy # define png_memset _fmemset -# define png_sprintf sprintf #else # ifdef _WINDOWS_ /* Favor Windows over C runtime fns */ # define CVT_PTR(ptr) (ptr) # define CVT_PTR_NOCHECK(ptr) (ptr) -# define png_strcpy lstrcpyA -# define png_strncpy lstrcpynA # define png_strlen lstrlenA # define png_memcmp memcmp # define png_memcpy CopyMemory # define png_memset memset -# define png_sprintf wsprintfA # else # define CVT_PTR(ptr) (ptr) # define CVT_PTR_NOCHECK(ptr) (ptr) -# define png_strcpy strcpy -# define png_strncpy strncpy /* Added to v 1.2.6 */ # define png_strlen strlen # define png_memcmp memcmp /* SJT: added */ # define png_memcpy memcpy # define png_memset memset -# define png_sprintf sprintf # endif #endif /* End of memory model/platform independent support */ - -#ifndef PNG_NO_SNPRINTF -# ifdef _MSC_VER -# define png_snprintf _snprintf /* Added to v 1.2.19 */ -# define png_snprintf2 _snprintf -# define png_snprintf6 _snprintf -# else -# define png_snprintf snprintf /* Added to v 1.2.19 */ -# define png_snprintf2 snprintf -# define png_snprintf6 snprintf -# endif -#else - /* You don't have or don't want to use snprintf(). Caution: Using - * sprintf instead of snprintf exposes your application to accidental - * or malevolent buffer overflows. If you don't have snprintf() - * as a general rule you should provide one (you can get one from - * Portable OpenSSH). - */ -# define png_snprintf(s1,n,fmt,x1) png_sprintf(s1,fmt,x1) -# define png_snprintf2(s1,n,fmt,x1,x2) png_sprintf(s1,fmt,x1,x2) -# define png_snprintf6(s1,n,fmt,x1,x2,x3,x4,x5,x6) \ - png_sprintf(s1,fmt,x1,x2,x3,x4,x5,x6) -#endif /* End of 1.5.0beta36 move from pngconf.h */ /* CONSTANTS and UTILITY MACROS @@ -261,12 +258,14 @@ typedef PNG_CONST png_uint_16p FAR * png_const_uint_16pp; */ /* Various modes of operation. Note that after an init, mode is set to - * zero automatically when the structure is created. + * zero automatically when the structure is created. Three of these + * are defined in png.h because they need to be visible to applications + * that call png_set_unknown_chunk(). */ -#define PNG_HAVE_IHDR 0x01 -#define PNG_HAVE_PLTE 0x02 +/* #define PNG_HAVE_IHDR 0x01 (defined in png.h) */ +/* #define PNG_HAVE_PLTE 0x02 (defined in png.h) */ #define PNG_HAVE_IDAT 0x04 -#define PNG_AFTER_IDAT 0x08 /* Have complete zlib datastream */ +/* #define PNG_AFTER_IDAT 0x08 (defined in png.h) */ #define PNG_HAVE_IEND 0x10 #define PNG_HAVE_gAMA 0x20 #define PNG_HAVE_cHRM 0x40 @@ -286,10 +285,10 @@ typedef PNG_CONST png_uint_16p FAR * png_const_uint_16pp; #define PNG_SWAP_BYTES 0x0010 #define PNG_INVERT_MONO 0x0020 #define PNG_QUANTIZE 0x0040 -#define PNG_BACKGROUND 0x0080 +#define PNG_COMPOSE 0x0080 /* Was PNG_BACKGROUND */ #define PNG_BACKGROUND_EXPAND 0x0100 #define PNG_EXPAND_16 0x0200 /* Added to libpng 1.5.2 */ -#define PNG_16_TO_8 0x0400 +#define PNG_16_TO_8 0x0400 /* Becomes 'chop' in 1.5.4 */ #define PNG_RGBA 0x0800 #define PNG_EXPAND 0x1000 #define PNG_GAMMA 0x2000 @@ -303,10 +302,10 @@ typedef PNG_CONST png_uint_16p FAR * png_const_uint_16pp; #define PNG_RGB_TO_GRAY_ERR 0x200000L #define PNG_RGB_TO_GRAY_WARN 0x400000L #define PNG_RGB_TO_GRAY 0x600000L /* two bits, RGB_TO_GRAY_ERR|WARN */ - /* 0x800000L Unused */ +#define PNG_ENCODE_ALPHA 0x800000L /* Added to libpng-1.5.4 */ #define PNG_ADD_ALPHA 0x1000000L /* Added to libpng-1.2.7 */ #define PNG_EXPAND_tRNS 0x2000000L /* Added to libpng-1.2.9 */ - /* 0x4000000L unused */ +#define PNG_SCALE_16_TO_8 0x4000000L /* Added to libpng-1.5.4 */ /* 0x8000000L unused */ /* 0x10000000L unused */ /* 0x20000000L unused */ @@ -333,9 +332,9 @@ typedef PNG_CONST png_uint_16p FAR * png_const_uint_16pp; #define PNG_FLAG_CRC_ANCILLARY_NOWARN 0x0200 #define PNG_FLAG_CRC_CRITICAL_USE 0x0400 #define PNG_FLAG_CRC_CRITICAL_IGNORE 0x0800 - /* 0x1000 unused */ - /* 0x2000 unused */ - /* 0x4000 unused */ +#define PNG_FLAG_ASSUME_sRGB 0x1000 /* Added to libpng-1.5.4 */ +#define PNG_FLAG_OPTIMIZE_ALPHA 0x2000 /* Added to libpng-1.5.4 */ +#define PNG_FLAG_DETECT_UNINITIALIZED 0x4000 /* Added to libpng-1.5.4 */ #define PNG_FLAG_KEEP_UNKNOWN_CHUNKS 0x8000L #define PNG_FLAG_KEEP_UNSAFE_CHUNKS 0x10000L #define PNG_FLAG_LIBRARY_MISMATCH 0x20000L @@ -345,13 +344,13 @@ typedef PNG_CONST png_uint_16p FAR * png_const_uint_16pp; /* 0x200000L unused */ /* 0x400000L unused */ #define PNG_FLAG_BENIGN_ERRORS_WARN 0x800000L /* Added to libpng-1.4.0 */ - /* 0x1000000L unused */ - /* 0x2000000L unused */ - /* 0x4000000L unused */ - /* 0x8000000L unused */ - /* 0x10000000L unused */ - /* 0x20000000L unused */ - /* 0x40000000L unused */ +#define PNG_FLAG_ZTXT_CUSTOM_STRATEGY 0x1000000L /* 5 lines added */ +#define PNG_FLAG_ZTXT_CUSTOM_LEVEL 0x2000000L /* to libpng-1.5.4 */ +#define PNG_FLAG_ZTXT_CUSTOM_MEM_LEVEL 0x4000000L +#define PNG_FLAG_ZTXT_CUSTOM_WINDOW_BITS 0x8000000L +#define PNG_FLAG_ZTXT_CUSTOM_METHOD 0x10000000L + /* 0x20000000L unused */ + /* 0x40000000L unused */ #define PNG_FLAG_CRC_ANCILLARY_MASK (PNG_FLAG_CRC_ANCILLARY_USE | \ PNG_FLAG_CRC_ANCILLARY_NOWARN) @@ -460,6 +459,11 @@ PNG_EXTERN png_fixed_point png_fixed PNGARG((png_structp png_ptr, double fp, #define PNG_tRNS PNG_CONST png_byte png_tRNS[5] = {116, 82, 78, 83, '\0'} #define PNG_zTXt PNG_CONST png_byte png_zTXt[5] = {122, 84, 88, 116, '\0'} +/* Gamma values (new at libpng-1.5.4): */ +#define PNG_GAMMA_MAC_OLD 151724 /* Assume '1.8' is really 2.2/1.45! */ +#define PNG_GAMMA_MAC_INVERSE 65909 +#define PNG_GAMMA_sRGB_INVERSE 45455 + /* Inhibit C++ name-mangling for libpng functions but not for system calls. */ #ifdef __cplusplus @@ -472,6 +476,12 @@ extern "C" { * be found in the files where the functions are located. */ +/* Check the user version string for compatibility, returns false if the version + * numbers aren't compatible. + */ +PNG_EXTERN int png_user_version_check(png_structp png_ptr, + png_const_charp user_png_ver); + /* Allocate memory for an internal libpng struct */ PNG_EXTERN PNG_FUNCTION(png_voidp,png_create_struct,PNGARG((int type)), PNG_ALLOCATED); @@ -540,8 +550,7 @@ PNG_EXTERN void png_crc_read PNGARG((png_structp png_ptr, png_bytep buf, png_size_t length)); /* Decompress data in a chunk that uses compression */ -#if defined(PNG_zTXt_SUPPORTED) || defined(PNG_iTXt_SUPPORTED) || \ - defined(PNG_iCCP_SUPPORTED) || defined(PNG_sPLT_SUPPORTED) +#if defined(PNG_READ_COMPRESSED_TEXT_SUPPORTED) PNG_EXTERN void png_decompress_chunk PNGARG((png_structp png_ptr, int comp_type, png_size_t chunklength, png_size_t prefix_length, png_size_t *data_length)); @@ -644,6 +653,7 @@ PNG_EXTERN void png_write_hIST PNGARG((png_structp png_ptr, png_const_uint_16p hist, int num_hist)); #endif +/* Chunks that have keywords */ #if defined(PNG_WRITE_TEXT_SUPPORTED) || defined(PNG_WRITE_pCAL_SUPPORTED) || \ defined(PNG_WRITE_iCCP_SUPPORTED) || defined(PNG_WRITE_sPLT_SUPPORTED) PNG_EXTERN png_size_t png_check_keyword PNGARG((png_structp png_ptr, @@ -734,17 +744,17 @@ PNG_EXTERN void png_read_filter_row PNGARG((png_structp png_ptr, PNG_EXTERN void png_write_find_filter PNGARG((png_structp png_ptr, png_row_infop row_info)); -/* Write out the filtered row. */ -PNG_EXTERN void png_write_filtered_row PNGARG((png_structp png_ptr, - png_bytep filtered_row)); /* Finish a row while reading, dealing with interlacing passes, etc. */ PNG_EXTERN void png_read_finish_row PNGARG((png_structp png_ptr)); /* Initialize the row buffers, etc. */ PNG_EXTERN void png_read_start_row PNGARG((png_structp png_ptr)); + +#ifdef PNG_READ_TRANSFORMS_SUPPORTED /* Optional call to update the users info structure */ PNG_EXTERN void png_read_transform_info PNGARG((png_structp png_ptr, png_infop info_ptr)); +#endif /* These are the functions that do the transformations */ #ifdef PNG_READ_FILLER_SUPPORTED @@ -816,7 +826,12 @@ PNG_EXTERN void png_do_invert PNGARG((png_row_infop row_info, png_bytep row)); #endif -#ifdef PNG_READ_16_TO_8_SUPPORTED +#ifdef PNG_READ_SCALE_16_TO_8_SUPPORTED +PNG_EXTERN void png_do_scale_16_to_8 PNGARG((png_row_infop row_info, + png_bytep row)); +#endif + +#ifdef PNG_READ_STRIP_16_TO_8_SUPPORTED PNG_EXTERN void png_do_chop PNGARG((png_row_infop row_info, png_bytep row)); #endif @@ -847,26 +862,20 @@ PNG_EXTERN void png_do_shift PNGARG((png_row_infop row_info, png_bytep row, png_const_color_8p bit_depth)); #endif -#ifdef PNG_READ_BACKGROUND_SUPPORTED -# ifdef PNG_READ_GAMMA_SUPPORTED -PNG_EXTERN void png_do_background PNGARG((png_row_infop row_info, - png_bytep row, png_const_color_16p trans_color, - png_const_color_16p background, png_const_color_16p background_1, - png_const_bytep gamma_table, png_const_bytep gamma_from_1, - png_const_bytep gamma_to_1, png_const_uint_16pp gamma_16, - png_const_uint_16pp gamma_16_from_1, png_const_uint_16pp gamma_16_to_1, - int gamma_shift)); -# else -PNG_EXTERN void png_do_background PNGARG((png_row_infop row_info, - png_bytep row, png_const_color_16p trans_color, - png_const_color_16p background)); -# endif +#if defined(PNG_READ_BACKGROUND_SUPPORTED) ||\ + defined(PNG_READ_ALPHA_MODE_SUPPORTED) +PNG_EXTERN void png_do_compose PNGARG((png_row_infop row_info, + png_bytep row, png_structp png_ptr)); #endif #ifdef PNG_READ_GAMMA_SUPPORTED PNG_EXTERN void png_do_gamma PNGARG((png_row_infop row_info, - png_bytep row, png_const_bytep gamma_table, - png_const_uint_16pp gamma_16_table, int gamma_shift)); + png_bytep row, png_structp png_ptr)); +#endif + +#ifdef PNG_READ_ALPHA_MODE_SUPPORTED +PNG_EXTERN void png_do_encode_alpha PNGARG((png_row_infop row_info, + png_bytep row, png_structp png_ptr)); #endif #ifdef PNG_READ_EXPAND_SUPPORTED @@ -986,10 +995,16 @@ PNG_EXTERN void png_check_chunk_name PNGARG((png_structp png_ptr, png_const_bytep chunk_name)); /* Handle the transformations for reading and writing */ +#ifdef PNG_READ_TRANSFORMS_SUPPORTED PNG_EXTERN void png_do_read_transformations PNGARG((png_structp png_ptr)); +#endif +#ifdef PNG_WRITE_TRANSFORMS_SUPPORTED PNG_EXTERN void png_do_write_transformations PNGARG((png_structp png_ptr)); +#endif +#ifdef PNG_READ_TRANSFORMS_SUPPORTED PNG_EXTERN void png_init_read_transformations PNGARG((png_structp png_ptr)); +#endif #ifdef PNG_PROGRESSIVE_READ_SUPPORTED PNG_EXTERN void png_push_read_chunk PNGARG((png_structp png_ptr, @@ -1086,6 +1101,76 @@ PNG_EXTERN PNG_FUNCTION(void, png_fixed_error, (png_structp png_ptr, png_const_charp name),PNG_NORETURN); #endif +/* Puts 'string' into 'buffer' at buffer[pos], taking care never to overwrite + * the end. Always leaves the buffer nul terminated. Never errors out (and + * there is no error code.) + */ +PNG_EXTERN size_t png_safecat(png_charp buffer, size_t bufsize, size_t pos, + png_const_charp string); + +/* Various internal functions to handle formatted warning messages, currently + * only implemented for warnings. + */ +#if defined(PNG_WARNINGS_SUPPORTED) || defined(PNG_TIME_RFC1123_SUPPORTED) +/* Utility to dump an unsigned value into a buffer, given a start pointer and + * and end pointer (which should point just *beyond* the end of the buffer!) + * Returns the pointer to the start of the formatted string. This utility only + * does unsigned values. + */ +PNG_EXTERN png_charp png_format_number(png_const_charp start, png_charp end, + int format, png_alloc_size_t number); + +/* Convenience macro that takes an array: */ +#define PNG_FORMAT_NUMBER(buffer,format,number) \ + png_format_number(buffer, buffer + (sizeof buffer), format, number) + +/* Suggested size for a number buffer (enough for 64 bits and a sign!) */ +#define PNG_NUMBER_BUFFER_SIZE 24 + +/* These are the integer formats currently supported, the name is formed from + * the standard printf(3) format string. + */ +#define PNG_NUMBER_FORMAT_u 1 /* chose unsigned API! */ +#define PNG_NUMBER_FORMAT_02u 2 +#define PNG_NUMBER_FORMAT_d 1 /* chose signed API! */ +#define PNG_NUMBER_FORMAT_02d 2 +#define PNG_NUMBER_FORMAT_x 3 +#define PNG_NUMBER_FORMAT_02x 4 +#define PNG_NUMBER_FORMAT_fixed 5 /* choose the signed API */ +#endif + +#ifdef PNG_WARNINGS_SUPPORTED +/* New defines and members adding in libpng-1.5.4 */ +# define PNG_WARNING_PARAMETER_SIZE 32 +# define PNG_WARNING_PARAMETER_COUNT 8 + +/* An l-value of this type has to be passed to the APIs below to cache the + * values of the parameters to a formatted warning message. + */ +typedef char png_warning_parameters[PNG_WARNING_PARAMETER_COUNT][ + PNG_WARNING_PARAMETER_SIZE]; + +PNG_EXTERN void png_warning_parameter(png_warning_parameters p, int number, + png_const_charp string); + /* Parameters are limited in size to PNG_WARNING_PARAMETER_SIZE characters, + * including the trailing '\0'. + */ +PNG_EXTERN void png_warning_parameter_unsigned(png_warning_parameters p, + int number, int format, png_alloc_size_t value); + /* Use png_alloc_size_t because it is an unsigned type as big as any we + * need to output. Use the following for a signed value. + */ +PNG_EXTERN void png_warning_parameter_signed(png_warning_parameters p, + int number, int format, png_int_32 value); + +PNG_EXTERN void png_formatted_warning(png_structp png_ptr, + png_warning_parameters p, png_const_charp message); + /* 'message' follows the X/Open approach of using @1, @2 to insert + * parameters previously supplied using the above functions. Errors in + * specifying the paramters will simple result in garbage substitutions. + */ +#endif + /* ASCII to FP interfaces, currently only implemented if sCAL * support is required. */ @@ -1148,8 +1233,18 @@ PNG_EXTERN void png_ascii_from_fixed PNGARG((png_structp png_ptr, #define PNG_FP_SAW_DOT 16 /* Saw a dot in current state */ #define PNG_FP_SAW_E 32 /* Saw an E (or e) in current state */ #define PNG_FP_SAW_ANY 60 /* Saw any of the above 4 */ + +/* These three values don't affect the parser. They are set but not used. + */ #define PNG_FP_WAS_VALID 64 /* Preceding substring is a valid fp number */ -#define PNG_FP_INVALID 128 /* Available for callers as a distinct value */ +#define PNG_FP_NEGATIVE 128 /* A negative number, including "-0" */ +#define PNG_FP_NONZERO 256 /* A non-zero value */ +#define PNG_FP_STICKY 448 /* The above three flags */ + +/* This is available for the caller to store in 'state' if required. Do not + * call the parser after setting it (the parser sometimes clears it.) + */ +#define PNG_FP_INVALID 512 /* Available for callers as a distinct value */ /* Result codes for the parser (boolean - true meants ok, false means * not ok yet.) @@ -1157,6 +1252,20 @@ PNG_EXTERN void png_ascii_from_fixed PNGARG((png_structp png_ptr, #define PNG_FP_MAYBE 0 /* The number may be valid in the future */ #define PNG_FP_OK 1 /* The number is valid */ +/* Tests on the sticky non-zero and negative flags. To pass these checks + * the state must also indicate that the whole number is valid - this is + * achieved by testing PNG_FP_SAW_DIGIT (see the implementation for why this + * is equivalent to PNG_FP_OK above.) + */ +#define PNG_FP_NZ_MASK (PNG_FP_SAW_DIGIT | PNG_FP_NEGATIVE | PNG_FP_NONZERO) + /* NZ_MASK: the string is valid and a non-zero negative value */ +#define PNG_FP_Z_MASK (PNG_FP_SAW_DIGIT | PNG_FP_NONZERO) + /* Z MASK: the string is valid and a non-zero value. */ + /* PNG_FP_SAW_DIGIT: the string is valid. */ +#define PNG_FP_IS_ZERO(state) (((state) & PNG_FP_Z_MASK) == PNG_FP_SAW_DIGIT) +#define PNG_FP_IS_POSITIVE(state) (((state) & PNG_FP_NZ_MASK) == PNG_FP_Z_MASK) +#define PNG_FP_IS_NEGATIVE(state) (((state) & PNG_FP_NZ_MASK) == PNG_FP_NZ_MASK) + /* The actual parser. This can be called repeatedly, it updates * the index into the string and the state variable (which must * be initialzed to 0). It returns a result code, as above. There @@ -1176,7 +1285,10 @@ PNG_EXTERN int png_check_fp_number PNGARG((png_const_charp string, png_size_t size, int *statep, png_size_tp whereami)); /* This is the same but it checks a complete string and returns true - * only if it just contains a floating point number. + * only if it just contains a floating point number. As of 1.5.4 this + * function also returns the state at the end of parsing the number if + * it was valid (otherwise it returns 0.) This can be used for testing + * for negative or zero values using the sticky flag. */ PNG_EXTERN int png_check_fp_string PNGARG((png_const_charp string, png_size_t size)); diff --git a/harbour/src/3rd/png/pngread.c b/harbour/src/3rd/png/pngread.c index aa84001c45..be3df47693 100644 --- a/harbour/src/3rd/png/pngread.c +++ b/harbour/src/3rd/png/pngread.c @@ -1,7 +1,7 @@ /* pngread.c - read a PNG file * - * Last changed in libpng 1.5.2 [March 31, 2011] + * Last changed in libpng 1.5.4 [July 7, 2011] * Copyright (c) 1998-2011 Glenn Randers-Pehrson * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) @@ -47,12 +47,10 @@ png_create_read_struct_2,(png_const_charp user_png_ver, png_voidp error_ptr, #ifdef PNG_SETJMP_SUPPORTED #ifdef USE_FAR_KEYWORD - jmp_buf png_jmpbuf; + jmp_buf tmp_jmpbuf; #endif #endif - int i; - png_debug(1, "in png_create_read_struct"); #ifdef PNG_USER_MEM_SUPPORTED @@ -85,13 +83,13 @@ png_create_read_struct_2,(png_const_charp user_png_ver, png_voidp error_ptr, encounter a png_error() will longjmp here. Since the jmpbuf is then meaningless we abort instead of returning. */ #ifdef USE_FAR_KEYWORD - if (setjmp(png_jmpbuf)) + if (setjmp(tmp_jmpbuf)) #else if (setjmp(png_jmpbuf(png_ptr))) /* Sets longjmp to match setjmp */ #endif PNG_ABORT(); #ifdef USE_FAR_KEYWORD - png_memcpy(png_jmpbuf(png_ptr), png_jmpbuf, png_sizeof(jmp_buf)); + png_memcpy(png_jmpbuf(png_ptr), tmp_jmpbuf, png_sizeof(jmp_buf)); #endif #endif /* PNG_SETJMP_SUPPORTED */ @@ -101,54 +99,9 @@ png_create_read_struct_2,(png_const_charp user_png_ver, png_voidp error_ptr, png_set_error_fn(png_ptr, error_ptr, error_fn, warn_fn); - if (user_png_ver) - { - i = 0; - - do - { - if (user_png_ver[i] != png_libpng_ver[i]) - png_ptr->flags |= PNG_FLAG_LIBRARY_MISMATCH; - } while (png_libpng_ver[i++]); - } - - else - png_ptr->flags |= PNG_FLAG_LIBRARY_MISMATCH; - - - if (png_ptr->flags & PNG_FLAG_LIBRARY_MISMATCH) - { - /* Libpng 0.90 and later are binary incompatible with libpng 0.89, so - * we must recompile any applications that use any older library version. - * For versions after libpng 1.0, we will be compatible, so we need - * only check the first digit. - */ - if (user_png_ver == NULL || user_png_ver[0] != png_libpng_ver[0] || - (user_png_ver[0] == '1' && user_png_ver[2] != png_libpng_ver[2]) || - (user_png_ver[0] == '0' && user_png_ver[2] < '9')) - { -#ifdef PNG_CONSOLE_IO_SUPPORTED - char msg[80]; - if (user_png_ver) - { - png_snprintf2(msg, 80, - "Application built with libpng-%.20s" - " but running with %.20s", - user_png_ver, - png_libpng_ver); - png_warning(png_ptr, msg); - } -#else - png_warning(png_ptr, - "Incompatible libpng version in application and library"); -#endif -#ifdef PNG_ERROR_NUMBERS_SUPPORTED - png_ptr->flags = 0; -#endif - - png_cleanup_needed = 1; - } - } + /* Call the general version checker (shared with read and write code): */ + if (!png_user_version_check(png_ptr, user_png_ver)) + png_cleanup_needed = 1; if (!png_cleanup_needed) { @@ -457,7 +410,11 @@ png_read_update_info(png_structp png_ptr, png_infop info_ptr) "Ignoring extra png_read_update_info() call;" " row buffer not reallocated"); +#ifdef PNG_READ_TRANSFORMS_SUPPORTED png_read_transform_info(png_ptr, info_ptr); +#else + PNG_UNUSED(info_ptr) +#endif } #ifdef PNG_SEQUENTIAL_READ_SUPPORTED @@ -704,8 +661,10 @@ png_read_row(png_structp png_ptr, png_bytep row, png_bytep dsp_row) #endif +#ifdef PNG_READ_TRANSFORMS_SUPPORTED if (png_ptr->transformations) png_do_read_transformations(png_ptr); +#endif #ifdef PNG_READ_INTERLACING_SUPPORTED /* Blow up interlaced rows to full size */ @@ -1163,7 +1122,9 @@ png_read_destroy(png_structp png_ptr, png_infop info_ptr, jmp_buf tmp_jmp; #endif png_error_ptr error_fn; +#ifdef PNG_WARNINGS_SUPPORTED png_error_ptr warning_fn; +#endif png_voidp error_ptr; #ifdef PNG_USER_MEM_SUPPORTED png_free_ptr free_fn; @@ -1249,10 +1210,6 @@ png_read_destroy(png_structp png_ptr, png_infop info_ptr, #endif #endif -#ifdef PNG_TIME_RFC1123_SUPPORTED - png_free(png_ptr, png_ptr->time_buffer); -#endif - inflateEnd(&png_ptr->zstream); #ifdef PNG_PROGRESSIVE_READ_SUPPORTED @@ -1269,11 +1226,13 @@ png_read_destroy(png_structp png_ptr, png_infop info_ptr, * being used again. */ #ifdef PNG_SETJMP_SUPPORTED - png_memcpy(tmp_jmp, png_ptr->png_jmpbuf, png_sizeof(jmp_buf)); + png_memcpy(tmp_jmp, png_ptr->longjmp_buffer, png_sizeof(jmp_buf)); #endif error_fn = png_ptr->error_fn; +#ifdef PNG_WARNINGS_SUPPORTED warning_fn = png_ptr->warning_fn; +#endif error_ptr = png_ptr->error_ptr; #ifdef PNG_USER_MEM_SUPPORTED free_fn = png_ptr->free_fn; @@ -1282,14 +1241,16 @@ png_read_destroy(png_structp png_ptr, png_infop info_ptr, png_memset(png_ptr, 0, png_sizeof(png_struct)); png_ptr->error_fn = error_fn; +#ifdef PNG_WARNINGS_SUPPORTED png_ptr->warning_fn = warning_fn; +#endif png_ptr->error_ptr = error_ptr; #ifdef PNG_USER_MEM_SUPPORTED png_ptr->free_fn = free_fn; #endif #ifdef PNG_SETJMP_SUPPORTED - png_memcpy(png_ptr->png_jmpbuf, tmp_jmp, png_sizeof(jmp_buf)); + png_memcpy(png_ptr->longjmp_buffer, tmp_jmp, png_sizeof(jmp_buf)); #endif } @@ -1325,8 +1286,22 @@ png_read_png(png_structp png_ptr, png_infop info_ptr, /* -------------- image transformations start here ------------------- */ -#ifdef PNG_READ_16_TO_8_SUPPORTED - /* Tell libpng to strip 16 bit/color files down to 8 bits per color. +#ifdef PNG_READ_SCALE_16_TO_8_SUPPORTED + /* Tell libpng to strip 16-bit/color files down to 8 bits per color. + */ + if (transforms & PNG_TRANSFORM_SCALE_16) + { + /* Added at libpng-1.5.4. "strip_16" produces the same result that it + * did in earlier versions, while "scale_16" is now more accurate. + */ + png_set_scale_16(png_ptr); + } +#endif + +#ifdef PNG_READ_STRIP_16_TO_8_SUPPORTED + /* If both SCALE and STRIP are required pngrtran will effectively cancel the + * latter by doing SCALE first. This is ok and allows apps not to check for + * which is supported to get the right answer. */ if (transforms & PNG_TRANSFORM_STRIP_16) png_set_strip_16(png_ptr); @@ -1407,7 +1382,7 @@ png_read_png(png_structp png_ptr, png_infop info_ptr, #endif #ifdef PNG_READ_SWAP_SUPPORTED - /* Swap bytes of 16 bit files to least significant byte first */ + /* Swap bytes of 16-bit files to least significant byte first */ if (transforms & PNG_TRANSFORM_SWAP_ENDIAN) png_set_swap(png_ptr); #endif @@ -1426,6 +1401,12 @@ png_read_png(png_structp png_ptr, png_infop info_ptr, png_set_gray_to_rgb(png_ptr); #endif +/* Added at libpng-1.5.4 */ +#ifdef PNG_READ_EXPAND_16_SUPPORTED + if (transforms & PNG_TRANSFORM_EXPAND_16) + png_set_expand_16(png_ptr); +#endif + /* We don't handle adding filler bytes */ /* We use png_read_image and rely on that for interlace handling, but we also diff --git a/harbour/src/3rd/png/pngrtran.c b/harbour/src/3rd/png/pngrtran.c index de5871add0..4e0401aa9f 100644 --- a/harbour/src/3rd/png/pngrtran.c +++ b/harbour/src/3rd/png/pngrtran.c @@ -1,7 +1,7 @@ /* pngrtran.c - transforms the data in a row for PNG readers * - * Last changed in libpng 1.5.2 [March 31, 2011] + * Last changed in libpng 1.5.4 [July 7, 2011] * Copyright (c) 1998-2011 Glenn Randers-Pehrson * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) @@ -106,12 +106,18 @@ png_set_background_fixed(png_structp png_ptr, return; } - png_ptr->transformations |= PNG_BACKGROUND; + png_ptr->transformations |= PNG_COMPOSE | PNG_STRIP_ALPHA; + png_ptr->transformations &= ~PNG_ENCODE_ALPHA; + png_ptr->flags &= ~PNG_FLAG_OPTIMIZE_ALPHA; + png_memcpy(&(png_ptr->background), background_color, png_sizeof(png_color_16)); png_ptr->background_gamma = background_gamma; png_ptr->background_gamma_type = (png_byte)(background_gamma_code); - png_ptr->transformations |= (need_expand ? PNG_BACKGROUND_EXPAND : 0); + if (need_expand) + png_ptr->transformations |= PNG_BACKGROUND_EXPAND; + else + png_ptr->transformations &= ~PNG_BACKGROUND_EXPAND; } # ifdef PNG_FLOATING_POINT_SUPPORTED @@ -126,8 +132,25 @@ png_set_background(png_structp png_ptr, # endif /* FLOATING_POINT */ #endif /* READ_BACKGROUND */ -#ifdef PNG_READ_16_TO_8_SUPPORTED -/* Strip 16 bit depth files to 8 bit depth */ +/* Scale 16-bit depth files to 8-bit depth. If both of these are set then the + * one that pngrtran does first (scale) happens. This is necessary to allow the + * TRANSFORM and API behavior to be somewhat consistent, and it's simpler. + */ +#ifdef PNG_READ_SCALE_16_TO_8_SUPPORTED +void PNGAPI +png_set_scale_16(png_structp png_ptr) +{ + png_debug(1, "in png_set_scale_16"); + + if (png_ptr == NULL) + return; + + png_ptr->transformations |= PNG_SCALE_16_TO_8; +} +#endif + +#ifdef PNG_READ_STRIP_16_TO_8_SUPPORTED +/* Chop 16-bit depth files to 8-bit depth */ void PNGAPI png_set_strip_16(png_structp png_ptr) { @@ -137,7 +160,6 @@ png_set_strip_16(png_structp png_ptr) return; png_ptr->transformations |= PNG_16_TO_8; - png_ptr->transformations &= ~PNG_EXPAND_16; } #endif @@ -154,8 +176,190 @@ png_set_strip_alpha(png_structp png_ptr) } #endif +#if defined(PNG_READ_ALPHA_MODE_SUPPORTED) || defined(PNG_READ_GAMMA_SUPPORTED) +static png_fixed_point +translate_gamma_flags(png_structp png_ptr, png_fixed_point output_gamma, + int is_screen) +{ + /* Check for flag values. The main reason for having the old Mac value as a + * flag is that it is pretty near impossible to work out what the correct + * value is from Apple documentation - a working Mac system is needed to + * discover the value! + */ + if (output_gamma == PNG_DEFAULT_sRGB || + output_gamma == PNG_FP_1 / PNG_DEFAULT_sRGB) + { + /* If there is no sRGB support this just sets the gamma to the standard + * sRGB value. (This is a side effect of using this function!) + */ +# ifdef PNG_READ_sRGB_SUPPORTED + png_ptr->flags |= PNG_FLAG_ASSUME_sRGB; +# endif + if (is_screen) + output_gamma = PNG_GAMMA_sRGB; + else + output_gamma = PNG_GAMMA_sRGB_INVERSE; + } + + else if (output_gamma == PNG_GAMMA_MAC_18 || + output_gamma == PNG_FP_1 / PNG_GAMMA_MAC_18) + { + if (is_screen) + output_gamma = PNG_GAMMA_MAC_OLD; + else + output_gamma = PNG_GAMMA_MAC_INVERSE; + } + + return output_gamma; +} + +# ifdef PNG_FLOATING_POINT_SUPPORTED +static png_fixed_point +convert_gamma_value(png_structp png_ptr, double output_gamma) +{ + /* The following silently ignores cases where fixed point (times 100,000) + * gamma values are passed to the floating point API. This is safe and it + * means the fixed point constants work just fine with the floating point + * API. The alternative would just lead to undetected errors and spurious + * bug reports. Negative values fail inside the _fixed API unless they + * correspond to the flag values. + */ + if (output_gamma > 0 && output_gamma < 128) + output_gamma *= PNG_FP_1; + + /* This preserves -1 and -2 exactly: */ + output_gamma = floor(output_gamma + .5); + + if (output_gamma > PNG_FP_MAX || output_gamma < PNG_FP_MIN) + png_fixed_error(png_ptr, "gamma value"); + + return (png_fixed_point)output_gamma; +} +# endif +#endif /* READ_ALPHA_MODE || READ_GAMMA */ + +#ifdef PNG_READ_ALPHA_MODE_SUPPORTED +void PNGFAPI +png_set_alpha_mode_fixed(png_structp png_ptr, int mode, + png_fixed_point output_gamma) +{ + int compose = 0; + png_fixed_point file_gamma; + + png_debug(1, "in png_set_alpha_mode"); + + if (png_ptr == NULL) + return; + + output_gamma = translate_gamma_flags(png_ptr, output_gamma, 1/*screen*/); + + /* Validate the value to ensure it is in a reasonable range. The value + * is expected to be 1 or greater, but this range test allows for some + * viewing correction values. The intent is to weed out users of this API + * who use the inverse of the gamma value accidentally! Since some of these + * values are reasonable this may have to be changed. + */ + if (output_gamma < 70000 || output_gamma > 300000) + png_error(png_ptr, "output gamma out of expected range"); + + /* The default file gamma is the inverse of the output gamma; the output + * gamma may be changed below so get the file value first: + */ + file_gamma = png_reciprocal(output_gamma); + + /* There are really 8 possibilities here, composed of any combination + * of: + * + * premultiply the color channels + * do not encode non-opaque pixels + * encode the alpha as well as the color channels + * + * The differences disappear if the input/output ('screen') gamma is 1.0, + * because then the encoding is a no-op and there is only the choice of + * premultiplying the color channels or not. + * + * png_set_alpha_mode and png_set_background interact because both use + * png_compose to do the work. Calling both is only useful when + * png_set_alpha_mode is used to set the default mode - PNG_ALPHA_PNG - along + * with a default gamma value. Otherwise PNG_COMPOSE must not be set. + */ + switch (mode) + { + case PNG_ALPHA_PNG: /* default: png standard */ + /* No compose, but it may be set by png_set_background! */ + png_ptr->transformations &= ~PNG_ENCODE_ALPHA; + png_ptr->flags &= ~PNG_FLAG_OPTIMIZE_ALPHA; + break; + + case PNG_ALPHA_ASSOCIATED: /* color channels premultiplied */ + compose = 1; + png_ptr->transformations &= ~PNG_ENCODE_ALPHA; + png_ptr->flags &= ~PNG_FLAG_OPTIMIZE_ALPHA; + /* The output is linear: */ + output_gamma = PNG_FP_1; + break; + + case PNG_ALPHA_OPTIMIZED: /* associated, non-opaque pixels linear */ + compose = 1; + png_ptr->transformations &= ~PNG_ENCODE_ALPHA; + png_ptr->flags |= PNG_FLAG_OPTIMIZE_ALPHA; + /* output_gamma records the encoding of opaque pixels! */ + break; + + case PNG_ALPHA_BROKEN: /* associated, non-linear, alpha encoded */ + compose = 1; + png_ptr->transformations |= PNG_ENCODE_ALPHA; + png_ptr->flags &= ~PNG_FLAG_OPTIMIZE_ALPHA; + break; + + default: + png_error(png_ptr, "invalid alpha mode"); + } + + /* Only set the default gamma if the file gamma has not been set (this has + * the side effect that the gamma in a second call to png_set_alpha_mode will + * be ignored.) + */ + if (png_ptr->gamma == 0) + png_ptr->gamma = file_gamma; + + /* But always set the output gamma: */ + png_ptr->screen_gamma = output_gamma; + + /* Finally, if pre-multiplying, set the background fields to achieve the + * desired result. + */ + if (compose) + { + /* And obtain alpha pre-multiplication by composing on black: */ + png_memset(&png_ptr->background, 0, sizeof png_ptr->background); + png_ptr->background_gamma = png_ptr->gamma; /* just in case */ + png_ptr->background_gamma_type = PNG_BACKGROUND_GAMMA_FILE; + png_ptr->transformations &= ~PNG_BACKGROUND_EXPAND; + + if (png_ptr->transformations & PNG_COMPOSE) + png_error(png_ptr, + "conflicting calls to set alpha mode and background"); + + png_ptr->transformations |= PNG_COMPOSE; + } + + /* New API, make sure apps call the correct initializers: */ + png_ptr->flags |= PNG_FLAG_DETECT_UNINITIALIZED; +} + +# ifdef PNG_FLOATING_POINT_SUPPORTED +void PNGAPI +png_set_alpha_mode(png_structp png_ptr, int mode, double output_gamma) +{ + png_set_alpha_mode_fixed(png_ptr, mode, convert_gamma_value(png_ptr, + output_gamma)); +} +# endif +#endif + #ifdef PNG_READ_QUANTIZE_SUPPORTED -/* Dither file to 8 bit. Supply a palette, the current number +/* Dither file to 8-bit. Supply a palette, the current number * of elements in the palette, the maximum number of elements * allowed, and a histogram if possible. If the current number * of colors is greater then the maximum number, the palette will be @@ -561,30 +765,6 @@ png_set_quantize(png_structp png_ptr, png_colorp palette, #endif /* PNG_READ_QUANTIZE_SUPPORTED */ #ifdef PNG_READ_GAMMA_SUPPORTED -/* Transform the image from the file_gamma to the screen_gamma. We - * only do transformations on images where the file_gamma and screen_gamma - * are not close reciprocals, otherwise it slows things down slightly, and - * also needlessly introduces small errors. - * - * We will turn off gamma transformation later if no semitransparent entries - * are present in the tRNS array for palette images. We can't do it here - * because we don't necessarily have the tRNS chunk yet. - */ -static int /* PRIVATE */ -png_gamma_threshold(png_fixed_point scrn_gamma, png_fixed_point file_gamma) -{ - /* PNG_GAMMA_THRESHOLD is the threshold for performing gamma - * correction as a difference of the overall transform from 1.0 - * - * We want to compare the threshold with s*f - 1, if we get - * overflow here it is because of wacky gamma values so we - * turn on processing anyway. - */ - png_fixed_point gtest; - return !png_muldiv(>est, scrn_gamma, file_gamma, PNG_FP_1) || - png_gamma_significant(gtest); -} - void PNGFAPI png_set_gamma_fixed(png_structp png_ptr, png_fixed_point scrn_gamma, png_fixed_point file_gamma) @@ -594,10 +774,32 @@ png_set_gamma_fixed(png_structp png_ptr, png_fixed_point scrn_gamma, if (png_ptr == NULL) return; - if ((png_ptr->color_type & PNG_COLOR_MASK_ALPHA) || - (png_ptr->color_type == PNG_COLOR_TYPE_PALETTE) || - png_gamma_threshold(scrn_gamma, file_gamma)) - png_ptr->transformations |= PNG_GAMMA; + /* New in libpng-1.5.4 - reserve particular negative values as flags. */ + scrn_gamma = translate_gamma_flags(png_ptr, scrn_gamma, 1/*screen*/); + file_gamma = translate_gamma_flags(png_ptr, file_gamma, 0/*file*/); + +#if PNG_LIBPNG_VER >= 10600 + /* Checking the gamma values for being >0 was added in 1.5.4 along with the + * premultiplied alpha support; this actually hides an undocumented feature + * of the previous implementation which allowed gamma processing to be + * disabled in background handling. There is no evidence (so far) that this + * was being used; however, png_set_background itself accepted and must still + * accept '0' for the gamma value it takes, because it isn't always used. + * + * Since this is an API change (albeit a very minor one that removes an + * undocumented API feature) it will not be made until libpng-1.6.0. + */ + if (file_gamma <= 0) + png_error(png_ptr, "invalid file gamma in png_set_gamma"); + + if (scrn_gamma <= 0) + png_error(png_ptr, "invalid screen gamma in png_set_gamma"); +#endif + + /* Set the gamma values unconditionally - this overrides the value in the PNG + * file if a gAMA chunk was present. png_set_alpha_mode provides a + * different, easier, way to default the file gamma. + */ png_ptr->gamma = file_gamma; png_ptr->screen_gamma = scrn_gamma; } @@ -606,9 +808,8 @@ png_set_gamma_fixed(png_structp png_ptr, png_fixed_point scrn_gamma, void PNGAPI png_set_gamma(png_structp png_ptr, double scrn_gamma, double file_gamma) { - png_set_gamma_fixed(png_ptr, - png_fixed(png_ptr, scrn_gamma, "png_set_gamma screen gamma"), - png_fixed(png_ptr, file_gamma, "png_set_gamma file gamma")); + png_set_gamma_fixed(png_ptr, convert_gamma_value(png_ptr, scrn_gamma), + convert_gamma_value(png_ptr, file_gamma)); } # endif /* FLOATING_POINT_SUPPORTED */ #endif /* READ_GAMMA */ @@ -688,7 +889,7 @@ png_set_tRNS_to_alpha(png_structp png_ptr) #endif /* defined(PNG_READ_EXPAND_SUPPORTED) */ #ifdef PNG_READ_EXPAND_16_SUPPORTED -/* Expand to 16 bit channels, expand the tRNS chunk too (because otherwise +/* Expand to 16-bit channels, expand the tRNS chunk too (because otherwise * it may not work correctly.) */ void PNGAPI @@ -700,9 +901,10 @@ png_set_expand_16(png_structp png_ptr) return; png_ptr->transformations |= (PNG_EXPAND_16 | PNG_EXPAND | PNG_EXPAND_tRNS); - png_ptr->transformations &= ~PNG_16_TO_8; - png_ptr->flags &= ~PNG_FLAG_ROW_INIT; + + /* New API, make sure apps call the correct initializers: */ + png_ptr->flags |= PNG_FLAG_DETECT_UNINITIALIZED; } #endif @@ -762,30 +964,38 @@ png_set_rgb_to_gray_fixed(png_structp png_ptr, int error_action, } #endif { - png_uint_16 red_int, green_int; - if (red < 0 || green < 0) + if (red >= 0 && green >= 0 && red + green <= PNG_FP_1) { - red_int = 6968; /* .212671 * 32768 + .5 */ - green_int = 23434; /* .715160 * 32768 + .5 */ - } + png_uint_16 red_int, green_int; - else if (red + green < 100000L) - { red_int = (png_uint_16)(((png_uint_32)red*32768L)/100000L); green_int = (png_uint_16)(((png_uint_32)green*32768L)/100000L); + + png_ptr->rgb_to_gray_red_coeff = red_int; + png_ptr->rgb_to_gray_green_coeff = green_int; + png_ptr->rgb_to_gray_blue_coeff = + (png_uint_16)(32768 - red_int - green_int); } else { - png_warning(png_ptr, "ignoring out of range rgb_to_gray coefficients"); - red_int = 6968; - green_int = 23434; - } + if (red >= 0 && green >= 0) + png_warning(png_ptr, + "ignoring out of range rgb_to_gray coefficients"); - png_ptr->rgb_to_gray_red_coeff = red_int; - png_ptr->rgb_to_gray_green_coeff = green_int; - png_ptr->rgb_to_gray_blue_coeff = - (png_uint_16)(32768 - red_int - green_int); + /* Use the defaults, from the cHRM chunk if set, else the built in Rec + * 709 values (which correspond to sRGB, so we don't have to worry + * about the sRGB chunk!) + */ + if (png_ptr->rgb_to_gray_red_coeff == 0 && + png_ptr->rgb_to_gray_green_coeff == 0 && + png_ptr->rgb_to_gray_blue_coeff == 0) + { + png_ptr->rgb_to_gray_red_coeff = 6968; /* .212671 * 32768 + .5 */ + png_ptr->rgb_to_gray_green_coeff = 23434; /* .715160 * 32768 + .5 */ + png_ptr->rgb_to_gray_blue_coeff = 2366; + } + } } } @@ -827,55 +1037,160 @@ png_set_read_user_transform_fn(png_structp png_ptr, png_user_transform_ptr } #endif +#ifdef PNG_READ_TRANSFORMS_SUPPORTED +#ifdef PNG_READ_GAMMA_SUPPORTED +/* In the case of gamma transformations only do transformations on images where + * the [file] gamma and screen_gamma are not close reciprocals, otherwise it + * slows things down slightly, and also needlessly introduces small errors. + */ +static int /* PRIVATE */ +png_gamma_threshold(png_fixed_point screen_gamma, png_fixed_point file_gamma) +{ + /* PNG_GAMMA_THRESHOLD is the threshold for performing gamma + * correction as a difference of the overall transform from 1.0 + * + * We want to compare the threshold with s*f - 1, if we get + * overflow here it is because of wacky gamma values so we + * turn on processing anyway. + */ + png_fixed_point gtest; + return !png_muldiv(>est, screen_gamma, file_gamma, PNG_FP_1) || + png_gamma_significant(gtest); +} +#endif + /* Initialize everything needed for the read. This includes modifying * the palette. */ -void /* PRIVATE */ -png_init_read_transformations(png_structp png_ptr) -{ - png_debug(1, "in png_init_read_transformations"); - { -#if defined(PNG_READ_BACKGROUND_SUPPORTED) || \ - defined(PNG_READ_SHIFT_SUPPORTED) || \ - defined(PNG_READ_GAMMA_SUPPORTED) - int color_type = png_ptr->color_type; -#endif +/*For the moment 'png_init_palette_transformations' and + * 'png_init_rgb_transformations' only do some flag canceling optimizations. + * The intent is that these two routines should have palette or rgb operations + * extracted from 'png_init_read_transformations'. + */ +static void /* PRIVATE */ +png_init_palette_transformations(png_structp png_ptr) +{ + /* Called to handle the (input) palette case. In png_do_read_transformations + * the first step is to expand the palette if requested, so this code must + * take care to only make changes that are invariant with respect to the + * palette expansion, or only do them if there is no expansion. + * + * STRIP_ALPHA has already been handled in the caller (by setting num_trans + * to 0.) + */ + int input_has_alpha = 0; + int input_has_transparency = 0; + + if (png_ptr->num_trans > 0) + { + int i; + + /* Ignore if all the entries are opaque (unlikely!) */ + for (i=0; inum_trans; ++i) + if (png_ptr->trans_alpha[i] == 255) + continue; + else if (png_ptr->trans_alpha[i] == 0) + input_has_transparency = 1; + else + input_has_alpha = 1; + } + + /* If no alpha we can optimize. */ + if (!input_has_alpha) + { + /* Any alpha means background and associative alpha processing is + * required, however if the alpha is 0 or 1 throughout OPTIIMIZE_ALPHA + * and ENCODE_ALPHA are irrelevant. + */ + png_ptr->transformations &= ~PNG_ENCODE_ALPHA; + png_ptr->flags &= ~PNG_FLAG_OPTIMIZE_ALPHA; + + if (!input_has_transparency) + png_ptr->transformations &= ~(PNG_COMPOSE | PNG_BACKGROUND_EXPAND); + } #if defined(PNG_READ_EXPAND_SUPPORTED) && defined(PNG_READ_BACKGROUND_SUPPORTED) - -#ifdef PNG_READ_GRAY_TO_RGB_SUPPORTED - /* Detect gray background and attempt to enable optimization - * for gray --> RGB case - * - * Note: if PNG_BACKGROUND_EXPAND is set and color_type is either RGB or - * RGB_ALPHA (in which case need_expand is superfluous anyway), the - * background color might actually be gray yet not be flagged as such. - * This is not a problem for the current code, which uses - * PNG_BACKGROUND_IS_GRAY only to decide when to do the - * png_do_gray_to_rgb() transformation. + /* png_set_background handling - deals with the complexity of whether the + * background color is in the file format or the screen format in the case + * where an 'expand' will happen. */ - if ((png_ptr->transformations & PNG_BACKGROUND_EXPAND) && - !(color_type & PNG_COLOR_MASK_COLOR)) - { - png_ptr->mode |= PNG_BACKGROUND_IS_GRAY; - } - - else if ((png_ptr->transformations & PNG_BACKGROUND) && - !(png_ptr->transformations & PNG_BACKGROUND_EXPAND) && - (png_ptr->transformations & PNG_GRAY_TO_RGB) && - png_ptr->background.red == png_ptr->background.green && - png_ptr->background.red == png_ptr->background.blue) - { - png_ptr->mode |= PNG_BACKGROUND_IS_GRAY; - png_ptr->background.gray = png_ptr->background.red; - } -#endif + /* The following code cannot be entered in the alpha pre-multiplication case + * because PNG_BACKGROUND_EXPAND is cancelled below. + */ if ((png_ptr->transformations & PNG_BACKGROUND_EXPAND) && (png_ptr->transformations & PNG_EXPAND)) { - if (!(color_type & PNG_COLOR_MASK_COLOR)) /* i.e., GRAY or GRAY_ALPHA */ + { + png_ptr->background.red = + png_ptr->palette[png_ptr->background.index].red; + png_ptr->background.green = + png_ptr->palette[png_ptr->background.index].green; + png_ptr->background.blue = + png_ptr->palette[png_ptr->background.index].blue; + +#ifdef PNG_READ_INVERT_ALPHA_SUPPORTED + if (png_ptr->transformations & PNG_INVERT_ALPHA) + { + if (!(png_ptr->transformations & PNG_EXPAND_tRNS)) + { + /* Invert the alpha channel (in tRNS) unless the pixels are + * going to be expanded, in which case leave it for later + */ + int i, istop = png_ptr->num_trans; + + for (i=0; itrans_alpha[i] = (png_byte)(255 - + png_ptr->trans_alpha[i]); + } + } +#endif /* PNG_READ_INVERT_ALPHA_SUPPORTED */ + } + } /* background expand and (therefore) no alpha association. */ +#endif /* PNG_READ_EXPAND_SUPPORTED && PNG_READ_BACKGROUND_SUPPORTED */ +} + +static void /* PRIVATE */ +png_init_rgb_transformations(png_structp png_ptr) +{ + /* Added to libpng-1.5.4: check the color type to determine whether there + * is any alpha or transparency in the image and simply cancel the + * background and alpha mode stuff if there isn't. + */ + int input_has_alpha = (png_ptr->color_type & PNG_COLOR_MASK_ALPHA) != 0; + int input_has_transparency = png_ptr->num_trans > 0; + + /* If no alpha we can optimize. */ + if (!input_has_alpha) + { + /* Any alpha means background and associative alpha processing is + * required, however if the alpha is 0 or 1 throughout OPTIIMIZE_ALPHA + * and ENCODE_ALPHA are irrelevant. + */ +# ifdef PNG_READ_ALPHA_MODE_SUPPORTED + png_ptr->transformations &= ~PNG_ENCODE_ALPHA; + png_ptr->flags &= ~PNG_FLAG_OPTIMIZE_ALPHA; +# endif + + if (!input_has_transparency) + png_ptr->transformations &= ~(PNG_COMPOSE | PNG_BACKGROUND_EXPAND); + } + +#if defined(PNG_READ_EXPAND_SUPPORTED) && defined(PNG_READ_BACKGROUND_SUPPORTED) + /* png_set_background handling - deals with the complexity of whether the + * background color is in the file format or the screen format in the case + * where an 'expand' will happen. + */ + + /* The following code cannot be entered in the alpha pre-multiplication case + * because PNG_BACKGROUND_EXPAND is cancelled below. + */ + if ((png_ptr->transformations & PNG_BACKGROUND_EXPAND) && + (png_ptr->transformations & PNG_EXPAND) && + !(png_ptr->color_type & PNG_COLOR_MASK_COLOR)) + /* i.e., GRAY or GRAY_ALPHA */ + { { /* Expand background and tRNS chunks */ switch (png_ptr->bit_depth) @@ -926,68 +1241,286 @@ png_init_read_transformations(png_structp png_ptr) break; } } - else if (color_type == PNG_COLOR_TYPE_PALETTE) - { - png_ptr->background.red = - png_ptr->palette[png_ptr->background.index].red; - png_ptr->background.green = - png_ptr->palette[png_ptr->background.index].green; - png_ptr->background.blue = - png_ptr->palette[png_ptr->background.index].blue; + } /* background expand and (therefore) no alpha association. */ +#endif /* PNG_READ_EXPAND_SUPPORTED && PNG_READ_BACKGROUND_SUPPORTED */ +} -#ifdef PNG_READ_INVERT_ALPHA_SUPPORTED - if (png_ptr->transformations & PNG_INVERT_ALPHA) - { -#ifdef PNG_READ_EXPAND_SUPPORTED - if (!(png_ptr->transformations & PNG_EXPAND_tRNS)) -#endif - { - /* Invert the alpha channel (in tRNS) unless the pixels are - * going to be expanded, in which case leave it for later - */ - int i, istop; - istop=(int)png_ptr->num_trans; - for (i=0; itrans_alpha[i] = (png_byte)(255 - - png_ptr->trans_alpha[i]); - } - } -#endif +void /* PRIVATE */ +png_init_read_transformations(png_structp png_ptr) +{ + png_debug(1, "in png_init_read_transformations"); - } - } -#endif + /* This internal function is called from png_read_start_row in pngrutil.c + * and it is called before the 'rowbytes' calculation is done, so the code + * in here can change or update the transformations flags. + * + * First do updates that do not depend on the details of the PNG image data + * being processed. + */ -#if defined(PNG_READ_BACKGROUND_SUPPORTED) && defined(PNG_READ_GAMMA_SUPPORTED) - png_ptr->background_1 = png_ptr->background; -#endif #ifdef PNG_READ_GAMMA_SUPPORTED - - if ((color_type == PNG_COLOR_TYPE_PALETTE && png_ptr->num_trans != 0) - && png_gamma_threshold(png_ptr->screen_gamma, png_ptr->gamma)) + /* Prior to 1.5.4 these tests were performed from png_set_gamma, 1.5.4 adds + * png_set_alpha_mode and this is another source for a default file gamma so + * the test needs to be performed later - here. In addition prior to 1.5.4 + * the tests were repeated for the PALETTE color type here - this is no + * longer necessary (and doesn't seem to have been necessary before.) + */ { - int i, k; - k=0; - for (i=0; inum_trans; i++) + /* The following temporary indicates if overall gamma correction is + * required. + */ + int gamma_correction = 0; + + if (png_ptr->gamma != 0) /* has been set */ { - if (png_ptr->trans_alpha[i] != 0 && png_ptr->trans_alpha[i] != 0xff) - k=1; /* Partial transparency is present */ + if (png_ptr->screen_gamma != 0) /* screen set too */ + gamma_correction = png_gamma_threshold(png_ptr->gamma, + png_ptr->screen_gamma); + + else + /* Assume the output matches the input; a long time default behavior + * of libpng, although the standard has nothing to say about this. + */ + png_ptr->screen_gamma = png_reciprocal(png_ptr->gamma); } - if (k == 0) + + else if (png_ptr->screen_gamma != 0) + /* The converse - assume the file matches the screen, note that this + * perhaps undesireable default can (from 1.5.4) be changed by calling + * png_set_alpha_mode (even if the alpha handling mode isn't required + * or isn't changed from the default.) + */ + png_ptr->gamma = png_reciprocal(png_ptr->screen_gamma); + + else /* neither are set */ + /* Just in case the following prevents any processing - file and screen + * are both assumed to be linear and there is no way to introduce a + * third gamma value other than png_set_background with 'UNIQUE', and, + * prior to 1.5.4 + */ + png_ptr->screen_gamma = png_ptr->gamma = PNG_FP_1; + + /* Now turn the gamma transformation on or off as appropriate. Notice + * that PNG_GAMMA just refers to the file->screen correction. Alpha + * composition may independently cause gamma correction because it needs + * linear data (e.g. if the file has a gAMA chunk but the screen gamma + * hasn't been specified.) In any case this flag may get turned off in + * the code immediately below if the transform can be handled outside the + * row loop. + */ + if (gamma_correction) + png_ptr->transformations |= PNG_GAMMA; + + else png_ptr->transformations &= ~PNG_GAMMA; } +#endif - if ((png_ptr->transformations & (PNG_GAMMA | PNG_RGB_TO_GRAY)) && - png_ptr->gamma != 0) + /* Certain transformations have the effect of preventing other + * transformations that happen afterward in png_do_read_transformations, + * resolve the interdependencies here. From the code of + * png_do_read_transformations the order is: + * + * 1) PNG_EXPAND (including PNG_EXPAND_tRNS) + * 2) PNG_STRIP_ALPHA (if no compose) + * 3) PNG_RGB_TO_GRAY + * 4) PNG_GRAY_TO_RGB iff !PNG_BACKGROUND_IS_GRAY + * 5) PNG_COMPOSE + * 6) PNG_GAMMA + * 7) PNG_STRIP_ALPHA (if compose) + * 8) PNG_ENCODE_ALPHA + * 9) PNG_SCALE_16_TO_8 + * 10) PNG_16_TO_8 + * 11) PNG_QUANTIZE (converts to palette) + * 12) PNG_EXPAND_16 + * 13) PNG_GRAY_TO_RGB iff PNG_BACKGROUND_IS_GRAY + * 14) PNG_INVERT_MONO + * 15) PNG_SHIFT + * 16) PNG_PACK + * 17) PNG_BGR + * 18) PNG_PACKSWAP + * 19) PNG_FILLER (includes PNG_ADD_ALPHA) + * 20) PNG_INVERT_ALPHA + * 21) PNG_SWAP_ALPHA + * 22) PNG_SWAP_BYTES + * 23) PNG_USER_TRANSFORM [must be last] + */ +#ifdef PNG_READ_STRIP_ALPHA_SUPPORTED + if ((png_ptr->transformations & PNG_STRIP_ALPHA) && + !(png_ptr->transformations & PNG_COMPOSE)) + { + /* Stripping the alpha channel happens immediately after the 'expand' + * transformations, before all other transformation, so it cancels out + * the alpha handling. It has the side effect negating the effect of + * PNG_EXPAND_tRNS too: + */ + png_ptr->transformations &= ~(PNG_BACKGROUND_EXPAND | PNG_ENCODE_ALPHA | + PNG_EXPAND_tRNS); + png_ptr->flags &= ~PNG_FLAG_OPTIMIZE_ALPHA; + + /* Kill the tRNS chunk itself too. Prior to 1.5.4 this did not happen + * so transparency information would remain just so long as it wasn't + * expanded. This produces unexpected API changes if the set of things + * that do PNG_EXPAND_tRNS changes (perfectly possible given the + * documentation - which says ask for what you want, accept what you + * get.) This makes the behavior consistent from 1.5.4: + */ + png_ptr->num_trans = 0; + } +#endif /* STRIP_ALPHA supported, no COMPOSE */ + +#ifdef PNG_READ_ALPHA_MODE_SUPPORTED + /* If the screen gamma is about 1.0 then the OPTIMIZE_ALPHA and ENCODE_ALPHA + * settings will have no effect. + */ + if (!png_gamma_significant(png_ptr->screen_gamma)) + { + png_ptr->transformations &= ~PNG_ENCODE_ALPHA; + png_ptr->flags &= ~PNG_FLAG_OPTIMIZE_ALPHA; + } +#endif + +#if defined(PNG_READ_EXPAND_SUPPORTED) && \ + defined(PNG_READ_BACKGROUND_SUPPORTED) && \ + defined(PNG_READ_GRAY_TO_RGB_SUPPORTED) + /* Detect gray background and attempt to enable optimization for + * gray --> RGB case. + * + * Note: if PNG_BACKGROUND_EXPAND is set and color_type is either RGB or + * RGB_ALPHA (in which case need_expand is superfluous anyway), the + * background color might actually be gray yet not be flagged as such. + * This is not a problem for the current code, which uses + * PNG_BACKGROUND_IS_GRAY only to decide when to do the + * png_do_gray_to_rgb() transformation. + * + * TODO: this code needs to be revised to avoid the complexity and + * interdependencies. The color type of the background should be recorded in + * png_set_background, along with the bit depth, then the code has a record + * of exactly what color space the background is currently in. + */ + if (png_ptr->transformations & PNG_BACKGROUND_EXPAND) + { + /* PNG_BACKGROUND_EXPAND: the background is in the file color space, so if + * the file was greyscale the background value is gray. + */ + if (!(png_ptr->color_type & PNG_COLOR_MASK_COLOR)) + png_ptr->mode |= PNG_BACKGROUND_IS_GRAY; + } + + else if (png_ptr->transformations & PNG_COMPOSE) + { + /* PNG_COMPOSE: png_set_background was called with need_expand false, + * so the color is in the color space of the output or png_set_alpha_mode + * was called and the color is black. Ignore RGB_TO_GRAY because that + * happens before GRAY_TO_RGB. + */ + if (png_ptr->transformations & PNG_GRAY_TO_RGB) + { + if (png_ptr->background.red == png_ptr->background.green && + png_ptr->background.red == png_ptr->background.blue) + { + png_ptr->mode |= PNG_BACKGROUND_IS_GRAY; + png_ptr->background.gray = png_ptr->background.red; + } + } + } +#endif /* PNG_READ_GRAY_TO_RGB_SUPPORTED (etc) */ + + /* For indexed PNG data (PNG_COLOR_TYPE_PALETTE) many of the transformations + * can be performed directly on the palette, and some (such as rgb to gray) + * can be optimized inside the palette. This is particularly true of the + * composite (background and alpha) stuff, which can be pretty much all done + * in the palette even if the result is expanded to RGB or gray afterward. + * + * NOTE: this is Not Yet Implemented, the code behaves as in 1.5.1 and + * earlier and the palette stuff is actually handled on the first row. This + * leads to the reported bug that the palette returned by png_get_PLTE is not + * updated. + */ + if (png_ptr->color_type == PNG_COLOR_TYPE_PALETTE) + png_init_palette_transformations(png_ptr); + + else + png_init_rgb_transformations(png_ptr); + +#if defined(PNG_READ_BACKGROUND_SUPPORTED) && \ + defined(PNG_READ_EXPAND_16_SUPPORTED) + if ((png_ptr->transformations & PNG_EXPAND_16) && + (png_ptr->transformations & PNG_COMPOSE) && + !(png_ptr->transformations & PNG_BACKGROUND_EXPAND) && + png_ptr->bit_depth != 16) + { + /* TODO: fix this. Because the expand_16 operation is after the compose + * handling the background color must be 8, not 16, bits deep, but the + * application will supply a 16-bit value so reduce it here. + * + * The PNG_BACKGROUND_EXPAND code above does not expand to 16 bits at + * present, so that case is ok (until do_expand_16 is moved.) + * + * NOTE: this discards the low 16 bits of the user supplied background + * color, but until expand_16 works properly there is no choice! + */ +# define CHOP(x) (x)=((png_uint_16)(((png_uint_32)(x)*255+32895) >> 16)) + CHOP(png_ptr->background.red); + CHOP(png_ptr->background.green); + CHOP(png_ptr->background.blue); + CHOP(png_ptr->background.gray); +# undef CHOP + } +#endif /* PNG_READ_BACKGROUND_SUPPORTED && PNG_READ_EXPAND_16_SUPPORTED */ + + /* NOTE: below 'PNG_READ_ALPHA_MODE_SUPPORTED' is presumed to also enable the + * background support (see the comments in scripts/pnglibconf.dfa), this + * allows pre-multiplication of the alpha channel to be implemented as + * compositing on black. This is probably sub-optimal and has been done in + * 1.5.4 betas simply to enable external critique and testing (i.e. to + * implement the new API quickly, without lots of internal changes.) + */ + +#ifdef PNG_READ_GAMMA_SUPPORTED +# ifdef PNG_READ_BACKGROUND_SUPPORTED + /* Includes ALPHA_MODE */ + png_ptr->background_1 = png_ptr->background; +# endif + + /* This needs to change - in the palette image case a whole set of tables are + * built when it would be quicker to just calculate the correct value for + * each palette entry directly. Also, the test is too tricky - why check + * PNG_RGB_TO_GRAY if PNG_GAMMA is not set? The answer seems to be that + * PNG_GAMMA is cancelled even if the gamma is known? The test excludes the + * PNG_COMPOSE case, so apparently if there is no *overall* gamma correction + * the gamma tables will not be built even if composition is required on a + * gamma encoded value. + * + * In 1.5.4 this is addressed below by an additional check on the individual + * file gamma - if it is not 1.0 both RGB_TO_GRAY and COMPOSE need the + * tables. + */ + if ((png_ptr->transformations & PNG_GAMMA) + || ((png_ptr->transformations & PNG_RGB_TO_GRAY) + && (png_gamma_significant(png_ptr->gamma) || + png_gamma_significant(png_ptr->screen_gamma))) + || ((png_ptr->transformations & PNG_COMPOSE) + && (png_gamma_significant(png_ptr->gamma) + || png_gamma_significant(png_ptr->screen_gamma) +# ifdef PNG_READ_BACKGROUND_SUPPORTED + || (png_ptr->background_gamma_type == PNG_BACKGROUND_GAMMA_UNIQUE + && png_gamma_significant(png_ptr->background_gamma)) +# endif + )) || ((png_ptr->transformations & PNG_ENCODE_ALPHA) + && png_gamma_significant(png_ptr->screen_gamma)) + ) { png_build_gamma_table(png_ptr, png_ptr->bit_depth); #ifdef PNG_READ_BACKGROUND_SUPPORTED - if (png_ptr->transformations & PNG_BACKGROUND) + if (png_ptr->transformations & PNG_COMPOSE) { - if (color_type == PNG_COLOR_TYPE_PALETTE) + if (png_ptr->color_type == PNG_COLOR_TYPE_PALETTE) { - /* Could skip if no transparency */ + /* We don't get to here unless there is a tRNS chunk with non-opaque + * entries - see the checking code at the start of this function. + */ png_color back, back_1; png_colorp palette = png_ptr->palette; int num_palette = png_ptr->num_palette; @@ -1032,13 +1565,6 @@ png_init_read_transformations(png_structp png_ptr) } if (png_gamma_significant(gs)) - { - back.red = (png_byte)png_ptr->background.red; - back.green = (png_byte)png_ptr->background.green; - back.blue = (png_byte)png_ptr->background.blue; - } - - else { back.red = png_gamma_8bit_correct(png_ptr->background.red, gs); @@ -1047,12 +1573,32 @@ png_init_read_transformations(png_structp png_ptr) back.blue = png_gamma_8bit_correct(png_ptr->background.blue, gs); } - back_1.red = png_gamma_8bit_correct(png_ptr->background.red, g); - back_1.green = png_gamma_8bit_correct(png_ptr->background.green, - g); - back_1.blue = png_gamma_8bit_correct(png_ptr->background.blue, - g); + + else + { + back.red = (png_byte)png_ptr->background.red; + back.green = (png_byte)png_ptr->background.green; + back.blue = (png_byte)png_ptr->background.blue; + } + + if (png_gamma_significant(g)) + { + back_1.red = png_gamma_8bit_correct(png_ptr->background.red, + g); + back_1.green = png_gamma_8bit_correct( + png_ptr->background.green, g); + back_1.blue = png_gamma_8bit_correct(png_ptr->background.blue, + g); + } + + else + { + back_1.red = (png_byte)png_ptr->background.red; + back_1.green = (png_byte)png_ptr->background.green; + back_1.blue = (png_byte)png_ptr->background.blue; + } } + for (i = 0; i < num_palette; i++) { if (i < (int)png_ptr->num_trans && @@ -1086,19 +1632,18 @@ png_init_read_transformations(png_structp png_ptr) palette[i].blue = png_ptr->gamma_table[palette[i].blue]; } } - /* Prevent the transformations being done again, and make sure - * that the now spurious alpha channel is stripped - the code - * has just reduced background composition and gamma correction - * to a simple alpha channel strip. + + /* Prevent the transformations being done again. + * + * NOTE: this is highly dubious; it zaps the transformations in + * place. This seems inconsistent with the general treatment of the + * transformations elsewhere. */ - png_ptr->transformations &= ~PNG_BACKGROUND; - png_ptr->transformations &= ~PNG_GAMMA; - png_ptr->transformations |= PNG_STRIP_ALPHA; - } + png_ptr->transformations &= ~(PNG_COMPOSE | PNG_GAMMA); + } /* color_type == PNG_COLOR_TYPE_PALETTE */ /* if (png_ptr->background_gamma_type!=PNG_BACKGROUND_GAMMA_UNKNOWN) */ - else - /* color_type != PNG_COLOR_TYPE_PALETTE */ + else /* color_type != PNG_COLOR_TYPE_PALETTE */ { png_fixed_point g = PNG_FP_1; png_fixed_point gs = PNG_FP_1; @@ -1164,17 +1709,21 @@ png_init_read_transformations(png_structp png_ptr) png_ptr->background.red = png_ptr->background.green = png_ptr->background.blue = png_ptr->background.gray; } - } - } + } /* color_type != PNG_COLOR_TYPE_PALETTE */ + }/* png_ptr->transformations & PNG_BACKGROUND */ + else /* Transformation does not include PNG_BACKGROUND */ #endif /* PNG_READ_BACKGROUND_SUPPORTED */ - if (color_type == PNG_COLOR_TYPE_PALETTE) + if (png_ptr->color_type == PNG_COLOR_TYPE_PALETTE) { png_colorp palette = png_ptr->palette; int num_palette = png_ptr->num_palette; int i; + /*NOTE: there are other transformations that should probably be in here + * too. + */ for (i = 0; i < num_palette; i++) { palette[i].red = png_ptr->gamma_table[palette[i].red]; @@ -1184,16 +1733,17 @@ png_init_read_transformations(png_structp png_ptr) /* Done the gamma correction. */ png_ptr->transformations &= ~PNG_GAMMA; - } + } /* color_type == PALETTE && !PNG_BACKGROUND transformation */ } #ifdef PNG_READ_BACKGROUND_SUPPORTED else #endif #endif /* PNG_READ_GAMMA_SUPPORTED */ + #ifdef PNG_READ_BACKGROUND_SUPPORTED - /* No GAMMA transformation */ - if ((png_ptr->transformations & PNG_BACKGROUND) && - (color_type == PNG_COLOR_TYPE_PALETTE)) + /* No GAMMA transformation (see the hanging else 4 lines above) */ + if ((png_ptr->transformations & PNG_COMPOSE) && + (png_ptr->color_type == PNG_COLOR_TYPE_PALETTE)) { int i; int istop = (int)png_ptr->num_trans; @@ -1225,15 +1775,13 @@ png_init_read_transformations(png_structp png_ptr) } } - /* Handled alpha, still need to strip the channel. */ - png_ptr->transformations &= ~PNG_BACKGROUND; - png_ptr->transformations |= PNG_STRIP_ALPHA; + png_ptr->transformations &= ~PNG_COMPOSE; } #endif /* PNG_READ_BACKGROUND_SUPPORTED */ #ifdef PNG_READ_SHIFT_SUPPORTED if ((png_ptr->transformations & PNG_SHIFT) && - (color_type == PNG_COLOR_TYPE_PALETTE)) + (png_ptr->color_type == PNG_COLOR_TYPE_PALETTE)) { png_uint_16 i; png_uint_16 istop = png_ptr->num_palette; @@ -1258,12 +1806,6 @@ png_init_read_transformations(png_structp png_ptr) } } #endif /* PNG_READ_SHIFT_SUPPORTED */ - } -#if !defined(PNG_READ_GAMMA_SUPPORTED) && !defined(PNG_READ_SHIFT_SUPPORTED) \ - && !defined(PNG_READ_BACKGROUND_SUPPORTED) - if (png_ptr) - return; -#endif } /* Modify the info structure to reflect the transformations. The @@ -1280,8 +1822,11 @@ png_read_transform_info(png_structp png_ptr, png_infop info_ptr) { if (info_ptr->color_type == PNG_COLOR_TYPE_PALETTE) { - if (png_ptr->num_trans && - (png_ptr->transformations & PNG_EXPAND_tRNS)) + /* This check must match what actually happens in + * png_do_expand_palette; if it ever checks the tRNS chunk to see if + * it is all opaque we must do the same (at present it does not.) + */ + if (png_ptr->num_trans > 0) info_ptr->color_type = PNG_COLOR_TYPE_RGB_ALPHA; else @@ -1305,44 +1850,61 @@ png_read_transform_info(png_structp png_ptr, png_infop info_ptr) } #endif -#ifdef PNG_READ_EXPAND_16_SUPPORTED - if (png_ptr->transformations & PNG_EXPAND_16 && info_ptr->bit_depth == 8 && - info_ptr->color_type != PNG_COLOR_TYPE_PALETTE) - { - info_ptr->bit_depth = 16; - } -#endif - -#ifdef PNG_READ_BACKGROUND_SUPPORTED - if (png_ptr->transformations & PNG_BACKGROUND) - { - info_ptr->color_type = (png_byte)(info_ptr->color_type & - ~PNG_COLOR_MASK_ALPHA); - info_ptr->num_trans = 0; +#if defined(PNG_READ_BACKGROUND_SUPPORTED) ||\ + defined(PNG_READ_ALPHA_MODE_SUPPORTED) + /* The following is almost certainly wrong unless the background value is in + * the screen space! + */ + if (png_ptr->transformations & PNG_COMPOSE) info_ptr->background = png_ptr->background; - } #endif #ifdef PNG_READ_GAMMA_SUPPORTED - if (png_ptr->transformations & PNG_GAMMA) - { - info_ptr->gamma = png_ptr->gamma; - } + /* The following used to be conditional on PNG_GAMMA (prior to 1.5.4), + * however it seems that the code in png_init_read_transformations, which has + * been called before this from png_read_update_info->png_read_start_row + * sometimes does the gamma transform and cancels the flag. + */ + info_ptr->gamma = png_ptr->gamma; #endif -#ifdef PNG_READ_16_TO_8_SUPPORTED -#ifdef PNG_READ_16BIT_SUPPORTED - if ((png_ptr->transformations & PNG_16_TO_8) && (info_ptr->bit_depth == 16)) - info_ptr->bit_depth = 8; -#else - /* Force chopping 16-bit input down to 8 */ if (info_ptr->bit_depth == 16) { - png_ptr->transformations |=PNG_16_TO_8; - info_ptr->bit_depth = 8; +# ifdef PNG_READ_16BIT_SUPPORTED +# ifdef PNG_READ_SCALE_16_TO_8_SUPPORTED + if (png_ptr->transformations & PNG_SCALE_16_TO_8) + info_ptr->bit_depth = 8; +# endif + +# ifdef PNG_READ_STRIP_16_TO_8_SUPPORTED + if (png_ptr->transformations & PNG_16_TO_8) + info_ptr->bit_depth = 8; +# endif + +# else + /* No 16 bit support: force chopping 16-bit input down to 8, in this case + * the app program can chose if both APIs are available by setting the + * correct scaling to use. + */ +# ifdef PNG_READ_STRIP_16_TO_8_SUPPORTED + /* For compatibility with previous versions use the strip method by + * default. This code works because if PNG_SCALE_16_TO_8 is already + * set the code below will do that in preference to the chop. + */ + png_ptr->transformations |= PNG_16_TO_8; + info_ptr->bit_depth = 8; +# else + +# if PNG_READ_SCALE_16_TO_8_SUPPORTED + png_ptr->transformations |= PNG_SCALE_16_TO_8; + info_ptr->bit_depth = 8; +# else + + CONFIGURATION ERROR: you must enable at least one 16 to 8 method +# endif +# endif +#endif /* !READ_16BIT_SUPPORTED */ } -#endif -#endif #ifdef PNG_READ_GRAY_TO_RGB_SUPPORTED if (png_ptr->transformations & PNG_GRAY_TO_RGB) @@ -1366,6 +1928,14 @@ png_read_transform_info(png_structp png_ptr, png_infop info_ptr) } #endif +#ifdef PNG_READ_EXPAND_16_SUPPORTED + if (png_ptr->transformations & PNG_EXPAND_16 && info_ptr->bit_depth == 8 && + info_ptr->color_type != PNG_COLOR_TYPE_PALETTE) + { + info_ptr->bit_depth = 16; + } +#endif + #ifdef PNG_READ_PACK_SUPPORTED if ((png_ptr->transformations & PNG_PACK) && (info_ptr->bit_depth < 8)) info_ptr->bit_depth = 8; @@ -1382,7 +1952,10 @@ png_read_transform_info(png_structp png_ptr, png_infop info_ptr) #ifdef PNG_READ_STRIP_ALPHA_SUPPORTED if (png_ptr->transformations & PNG_STRIP_ALPHA) + { info_ptr->color_type &= ~PNG_COLOR_MASK_ALPHA; + info_ptr->num_trans = 0; + } #endif if (info_ptr->color_type & PNG_COLOR_MASK_ALPHA) @@ -1418,6 +1991,14 @@ defined(PNG_READ_USER_TRANSFORM_SUPPORTED) info_ptr->rowbytes = PNG_ROWBYTES(info_ptr->pixel_depth, info_ptr->width); + /* Adding in 1.5.4: cache the above value in png_struct so that we can later + * check in png_rowbytes that the user buffer won't get overwritten. Note + * that the field is not always set - if png_read_update_info isn't called + * the application has to either not do any transforms or get the calculation + * right itself. + */ + png_ptr->info_rowbytes = info_ptr->rowbytes; + #ifndef PNG_READ_EXPAND_SUPPORTED if (png_ptr) return; @@ -1435,29 +2016,27 @@ png_do_read_transformations(png_structp png_ptr) if (png_ptr->row_buf == NULL) { -#ifdef PNG_CONSOLE_IO_SUPPORTED - char msg[50]; - - png_snprintf2(msg, 50, - "NULL row buffer for row %ld, pass %d", (long)png_ptr->row_number, - png_ptr->pass); - png_error(png_ptr, msg); -#else - png_error(png_ptr, "NULL row buffer"); -#endif - } -#ifdef PNG_WARN_UNINITIALIZED_ROW - if (!(png_ptr->flags & PNG_FLAG_ROW_INIT)) - /* Application has failed to call either png_read_start_image() - * or png_read_update_info() after setting transforms that expand - * pixels. This check added to libpng-1.2.19 + /* Prior to 1.5.4 this output row/pass where the NULL pointer is, but this + * error is incredibly rare and incredibly easy to debug without this + * information. + */ + png_error(png_ptr, "NULL row buffer"); + } + + /* The following is debugging; prior to 1.5.4 the code was never compiled in; + * in 1.5.4 PNG_FLAG_DETECT_UNINITIALIZED was added and the macro + * PNG_WARN_UNINITIALIZED_ROW removed. In 1.5 the new flag is set only for + * selected new APIs to ensure that there is no API change. + */ + if ((png_ptr->flags & PNG_FLAG_DETECT_UNINITIALIZED) != 0 && + !(png_ptr->flags & PNG_FLAG_ROW_INIT)) + { + /* Application has failed to call either png_read_start_image() or + * png_read_update_info() after setting transforms that expand pixels. + * This check added to libpng-1.2.19 (but not enabled until 1.5.4). */ -#if (PNG_WARN_UNINITIALIZED_ROW==1) png_error(png_ptr, "Uninitialized row"); -#else - png_warning(png_ptr, "Uninitialized row"); -#endif -#endif + } #ifdef PNG_READ_EXPAND_SUPPORTED if (png_ptr->transformations & PNG_EXPAND) @@ -1482,16 +2061,13 @@ png_do_read_transformations(png_structp png_ptr) } #endif - /* Delay the 'expand 16' step until later for efficiency, so that the - * intermediate steps work with 8 bit data. - */ - #ifdef PNG_READ_STRIP_ALPHA_SUPPORTED if ((png_ptr->transformations & PNG_STRIP_ALPHA) && + !(png_ptr->transformations & PNG_COMPOSE) && (png_ptr->row_info.color_type == PNG_COLOR_TYPE_RGB_ALPHA || png_ptr->row_info.color_type == PNG_COLOR_TYPE_GRAY_ALPHA)) png_do_strip_channel(&(png_ptr->row_info), png_ptr->row_buf + 1, - 0/*!at_start, because SWAP_ALPHA happens later*/); + 0 /* at_start == false, because SWAP_ALPHA happens later */); #endif #ifdef PNG_READ_RGB_TO_GRAY_SUPPORTED @@ -1555,36 +2131,49 @@ png_do_read_transformations(png_structp png_ptr) png_do_gray_to_rgb(&(png_ptr->row_info), png_ptr->row_buf + 1); #endif -#ifdef PNG_READ_BACKGROUND_SUPPORTED - if ((png_ptr->transformations & PNG_BACKGROUND) && - ((png_ptr->num_trans != 0) || - (png_ptr->color_type & PNG_COLOR_MASK_ALPHA))) - png_do_background(&(png_ptr->row_info), png_ptr->row_buf + 1, - &(png_ptr->trans_color), &(png_ptr->background) -#ifdef PNG_READ_GAMMA_SUPPORTED - , &(png_ptr->background_1), - png_ptr->gamma_table, png_ptr->gamma_from_1, - png_ptr->gamma_to_1, png_ptr->gamma_16_table, - png_ptr->gamma_16_from_1, png_ptr->gamma_16_to_1, - png_ptr->gamma_shift -#endif - ); +#if (defined PNG_READ_BACKGROUND_SUPPORTED) ||\ + (defined PNG_READ_ALPHA_MODE_SUPPORTED) + if (png_ptr->transformations & PNG_COMPOSE) + png_do_compose(&(png_ptr->row_info), png_ptr->row_buf + 1, png_ptr); #endif #ifdef PNG_READ_GAMMA_SUPPORTED if ((png_ptr->transformations & PNG_GAMMA) && -#ifdef PNG_READ_BACKGROUND_SUPPORTED - !((png_ptr->transformations & PNG_BACKGROUND) && +#if (defined PNG_READ_BACKGROUND_SUPPORTED) ||\ + (defined PNG_READ_ALPHA_MODE_SUPPORTED) + !((png_ptr->transformations & PNG_COMPOSE) && ((png_ptr->num_trans != 0) || (png_ptr->color_type & PNG_COLOR_MASK_ALPHA))) && #endif (png_ptr->color_type != PNG_COLOR_TYPE_PALETTE)) - png_do_gamma(&(png_ptr->row_info), png_ptr->row_buf + 1, - png_ptr->gamma_table, png_ptr->gamma_16_table, - png_ptr->gamma_shift); + png_do_gamma(&(png_ptr->row_info), png_ptr->row_buf + 1, png_ptr); #endif -#ifdef PNG_READ_16_TO_8_SUPPORTED +#ifdef PNG_READ_STRIP_ALPHA_SUPPORTED + if ((png_ptr->transformations & PNG_STRIP_ALPHA) && + (png_ptr->transformations & PNG_COMPOSE) && + (png_ptr->row_info.color_type == PNG_COLOR_TYPE_RGB_ALPHA || + png_ptr->row_info.color_type == PNG_COLOR_TYPE_GRAY_ALPHA)) + png_do_strip_channel(&(png_ptr->row_info), png_ptr->row_buf + 1, + 0 /* at_start == false, because SWAP_ALPHA happens later */); +#endif + +#ifdef PNG_READ_ALPHA_MODE_SUPPORTED + if ((png_ptr->transformations & PNG_ENCODE_ALPHA) && + (png_ptr->row_info.color_type & PNG_COLOR_MASK_ALPHA)) + png_do_encode_alpha(&(png_ptr->row_info), png_ptr->row_buf + 1, png_ptr); +#endif + +#ifdef PNG_READ_SCALE_16_TO_8_SUPPORTED + if (png_ptr->transformations & PNG_SCALE_16_TO_8) + png_do_scale_16_to_8(&(png_ptr->row_info), png_ptr->row_buf + 1); +#endif + +#ifdef PNG_READ_STRIP_16_TO_8_SUPPORTED + /* There is no harm in doing both of these because only one has any effect, + * by putting the 'scale' option first if the app asks for scale (either by + * calling the API or in a TRANSFORM flag) this is what happens. + */ if (png_ptr->transformations & PNG_16_TO_8) png_do_chop(&(png_ptr->row_info), png_ptr->row_buf + 1); #endif @@ -1610,6 +2199,13 @@ png_do_read_transformations(png_structp png_ptr) png_do_expand_16(&png_ptr->row_info, png_ptr->row_buf + 1); #endif +#ifdef PNG_READ_GRAY_TO_RGB_SUPPORTED + /*NOTE: moved here in 1.5.4 (from much later in this list.) */ + if ((png_ptr->transformations & PNG_GRAY_TO_RGB) && + (png_ptr->mode & PNG_BACKGROUND_IS_GRAY)) + png_do_gray_to_rgb(&(png_ptr->row_info), png_ptr->row_buf + 1); +#endif + #ifdef PNG_READ_INVERT_SUPPORTED if (png_ptr->transformations & PNG_INVERT_MONO) png_do_invert(&(png_ptr->row_info), png_ptr->row_buf + 1); @@ -1636,16 +2232,6 @@ png_do_read_transformations(png_structp png_ptr) png_do_packswap(&(png_ptr->row_info), png_ptr->row_buf + 1); #endif -#ifdef PNG_READ_GRAY_TO_RGB_SUPPORTED - /*NOTE: this must be in the wrong place - what happens if BGR is set too? - * Need pngvalid to test this combo. - */ - /* If gray -> RGB, do so now only if we did not do so above */ - if ((png_ptr->transformations & PNG_GRAY_TO_RGB) && - (png_ptr->mode & PNG_BACKGROUND_IS_GRAY)) - png_do_gray_to_rgb(&(png_ptr->row_info), png_ptr->row_buf + 1); -#endif - #ifdef PNG_READ_FILLER_SUPPORTED if (png_ptr->transformations & PNG_FILLER) png_do_read_filler(&(png_ptr->row_info), png_ptr->row_buf + 1, @@ -1697,7 +2283,6 @@ png_do_read_transformations(png_structp png_ptr) png_ptr->row_info.width); } #endif - } #ifdef PNG_READ_PACK_SUPPORTED @@ -1919,54 +2504,86 @@ png_do_unshift(png_row_infop row_info, png_bytep row, } #endif -#ifdef PNG_READ_16_TO_8_SUPPORTED -/* Chop rows of bit depth 16 down to 8 */ +#ifdef PNG_READ_SCALE_16_TO_8_SUPPORTED +/* Scale rows of bit depth 16 down to 8 accurately */ void /* PRIVATE */ +png_do_scale_16_to_8(png_row_infop row_info, png_bytep row) +{ + png_debug(1, "in png_do_scale_16_to_8"); + + if (row_info->bit_depth == 16) + { + png_bytep sp = row; /* source */ + png_bytep dp = row; /* destinaton */ + png_bytep ep = sp + row_info->rowbytes; /* end+1 */ + + while (sp < ep) + { + /* The input is an array of 16 bit components, these must be scaled to + * 8 bits each. For a 16 bit value V the required value (from the PNG + * specification) is: + * + * (V * 255) / 65535 + * + * This reduces to round(V / 257), or floor((V + 128.5)/257) + * + * Represent V as the two byte value vhi.vlo. Make a guess that the + * result is the top byte of V, vhi, then the correction to this value + * is: + * + * error = floor(((V-vhi.vhi) + 128.5) / 257) + * = floor(((vlo-vhi) + 128.5) / 257) + * + * This can be approximated using integer arithmetic (and a signed + * shift): + * + * error = (vlo-vhi+128) >> 8; + * + * The approximate differs from the exact answer only when (vlo-vhi) is + * 128; it then gives a correction of +1 when the exact correction is + * 0. This gives 128 errors. The exact answer (correct for all 16 bit + * input values) is: + * + * error = (vlo-vhi+128)*65535 >> 24; + * + * An alternative arithmetic calculation which also gives no errors is: + * + * (V * 255 + 32895) >> 16 + */ + + png_int_32 tmp = *sp++; /* must be signed! */ + tmp += (((int)*sp++ - tmp + 128) * 65535) >> 24; + *dp++ = (png_byte)tmp; + } + + row_info->bit_depth = 8; + row_info->pixel_depth = (png_byte)(8 * row_info->channels); + row_info->rowbytes = row_info->width * row_info->channels; + } +} +#endif + +#ifdef PNG_READ_STRIP_16_TO_8_SUPPORTED +void /* PRIVATE */ +/* Simply discard the low byte. This was the default behavior prior + * to libpng-1.5.4. + */ png_do_chop(png_row_infop row_info, png_bytep row) { png_debug(1, "in png_do_chop"); if (row_info->bit_depth == 16) { - png_bytep sp = row; - png_bytep dp = row; - png_uint_32 i; - png_uint_32 istop = row_info->width * row_info->channels; + png_bytep sp = row; /* source */ + png_bytep dp = row; /* destinaton */ + png_bytep ep = sp + row_info->rowbytes; /* end+1 */ - for (i = 0; i> 8)) >> 8; - * - * Approximate calculation with shift/add instead of multiply/divide: - * *dp = ((((png_uint_32)(*sp) << 8) | - * (png_uint_32)((int)(*(sp + 1)) - *sp)) + 128) >> 8; - * - * What we actually do to avoid extra shifting and conversion: - */ - - *dp = *sp + ((((int)(*(sp + 1)) - *sp) > 128) ? 1 : 0); -#else - /* Simply discard the low order byte */ - *dp = *sp; -#endif + *dp++ = *sp; + sp += 2; /* skip low byte */ } + row_info->bit_depth = 8; row_info->pixel_depth = (png_byte)(8 * row_info->channels); row_info->rowbytes = row_info->width * row_info->channels; @@ -2721,11 +3338,13 @@ png_do_rgb_to_gray(png_structp png_ptr, png_row_infop row_info, png_bytep row) return rgb_error; } #endif +#endif /* PNG_READ_TRANSFORMS_SUPPORTED */ +#ifdef PNG_BUILD_GRAYSCALE_PALETTE_SUPPORTED /* Build a grayscale palette. Palette is assumed to be 1 << bit_depth * large of png_color. This lets grayscale images be treated as * paletted. Most useful for gamma correction and simplification - * of code. + * of code. This API is not used internally. */ void PNGAPI png_build_grayscale_palette(int bit_depth, png_colorp palette) @@ -2775,34 +3394,36 @@ png_build_grayscale_palette(int bit_depth, png_colorp palette) palette[i].blue = (png_byte)v; } } +#endif +#ifdef PNG_READ_TRANSFORMS_SUPPORTED #ifdef PNG_READ_BACKGROUND_SUPPORTED /* Replace any alpha or transparency with the supplied background color. * "background" is already in the screen gamma, while "background_1" is * at a gamma of 1.0. Paletted files have already been taken care of. */ void /* PRIVATE */ -png_do_background(png_row_infop row_info, png_bytep row, - png_const_color_16p trans_color, png_const_color_16p background -#ifdef PNG_READ_GAMMA_SUPPORTED - , png_const_color_16p background_1, png_const_bytep gamma_table, - png_const_bytep gamma_from_1, png_const_bytep gamma_to_1, - png_const_uint_16pp gamma_16, png_const_uint_16pp gamma_16_from_1, - png_const_uint_16pp gamma_16_to_1, int gamma_shift -#endif - ) +png_do_compose(png_row_infop row_info, png_bytep row, png_structp png_ptr) { - png_bytep sp, dp; +#ifdef PNG_READ_GAMMA_SUPPORTED + png_const_bytep gamma_table = png_ptr->gamma_table; + png_const_bytep gamma_from_1 = png_ptr->gamma_from_1; + png_const_bytep gamma_to_1 = png_ptr->gamma_to_1; + png_const_uint_16pp gamma_16 = png_ptr->gamma_16_table; + png_const_uint_16pp gamma_16_from_1 = png_ptr->gamma_16_from_1; + png_const_uint_16pp gamma_16_to_1 = png_ptr->gamma_16_to_1; + int gamma_shift = png_ptr->gamma_shift; +#endif + + png_bytep sp; png_uint_32 i; png_uint_32 row_width = row_info->width; + int optimize = (png_ptr->flags & PNG_FLAG_OPTIMIZE_ALPHA) != 0; int shift; - png_debug(1, "in png_do_background"); + png_debug(1, "in png_do_compose"); - if (background != NULL && - (!(row_info->color_type & PNG_COLOR_MASK_ALPHA) || - (row_info->color_type != PNG_COLOR_TYPE_PALETTE && trans_color))) { switch (row_info->color_type) { @@ -2817,10 +3438,10 @@ png_do_background(png_row_infop row_info, png_bytep row, for (i = 0; i < row_width; i++) { if ((png_uint_16)((*sp >> shift) & 0x01) - == trans_color->gray) + == png_ptr->trans_color.gray) { *sp &= (png_byte)((0x7f7f >> (7 - shift)) & 0xff); - *sp |= (png_byte)(background->gray << shift); + *sp |= (png_byte)(png_ptr->background.gray << shift); } if (!shift) @@ -2845,10 +3466,10 @@ png_do_background(png_row_infop row_info, png_bytep row, for (i = 0; i < row_width; i++) { if ((png_uint_16)((*sp >> shift) & 0x03) - == trans_color->gray) + == png_ptr->trans_color.gray) { *sp &= (png_byte)((0x3f3f >> (6 - shift)) & 0xff); - *sp |= (png_byte)(background->gray << shift); + *sp |= (png_byte)(png_ptr->background.gray << shift); } else @@ -2879,10 +3500,10 @@ png_do_background(png_row_infop row_info, png_bytep row, for (i = 0; i < row_width; i++) { if ((png_uint_16)((*sp >> shift) & 0x03) - == trans_color->gray) + == png_ptr->trans_color.gray) { *sp &= (png_byte)((0x3f3f >> (6 - shift)) & 0xff); - *sp |= (png_byte)(background->gray << shift); + *sp |= (png_byte)(png_ptr->background.gray << shift); } if (!shift) @@ -2908,10 +3529,10 @@ png_do_background(png_row_infop row_info, png_bytep row, for (i = 0; i < row_width; i++) { if ((png_uint_16)((*sp >> shift) & 0x0f) - == trans_color->gray) + == png_ptr->trans_color.gray) { *sp &= (png_byte)((0xf0f >> (4 - shift)) & 0xff); - *sp |= (png_byte)(background->gray << shift); + *sp |= (png_byte)(png_ptr->background.gray << shift); } else @@ -2942,10 +3563,10 @@ png_do_background(png_row_infop row_info, png_bytep row, for (i = 0; i < row_width; i++) { if ((png_uint_16)((*sp >> shift) & 0x0f) - == trans_color->gray) + == png_ptr->trans_color.gray) { *sp &= (png_byte)((0xf0f >> (4 - shift)) & 0xff); - *sp |= (png_byte)(background->gray << shift); + *sp |= (png_byte)(png_ptr->background.gray << shift); } if (!shift) @@ -2969,8 +3590,8 @@ png_do_background(png_row_infop row_info, png_bytep row, sp = row; for (i = 0; i < row_width; i++, sp++) { - if (*sp == trans_color->gray) - *sp = (png_byte)background->gray; + if (*sp == png_ptr->trans_color.gray) + *sp = (png_byte)png_ptr->background.gray; else *sp = gamma_table[*sp]; @@ -2982,8 +3603,8 @@ png_do_background(png_row_infop row_info, png_bytep row, sp = row; for (i = 0; i < row_width; i++, sp++) { - if (*sp == trans_color->gray) - *sp = (png_byte)background->gray; + if (*sp == png_ptr->trans_color.gray) + *sp = (png_byte)png_ptr->background.gray; } } break; @@ -3001,11 +3622,11 @@ png_do_background(png_row_infop row_info, png_bytep row, v = (png_uint_16)(((*sp) << 8) + *(sp + 1)); - if (v == trans_color->gray) + if (v == png_ptr->trans_color.gray) { /* Background is already in screen gamma */ - *sp = (png_byte)((background->gray >> 8) & 0xff); - *(sp + 1) = (png_byte)(background->gray & 0xff); + *sp = (png_byte)((png_ptr->background.gray >> 8) & 0xff); + *(sp + 1) = (png_byte)(png_ptr->background.gray & 0xff); } else @@ -3026,10 +3647,10 @@ png_do_background(png_row_infop row_info, png_bytep row, v = (png_uint_16)(((*sp) << 8) + *(sp + 1)); - if (v == trans_color->gray) + if (v == png_ptr->trans_color.gray) { - *sp = (png_byte)((background->gray >> 8) & 0xff); - *(sp + 1) = (png_byte)(background->gray & 0xff); + *sp = (png_byte)((png_ptr->background.gray >> 8) & 0xff); + *(sp + 1) = (png_byte)(png_ptr->background.gray & 0xff); } } } @@ -3052,13 +3673,13 @@ png_do_background(png_row_infop row_info, png_bytep row, sp = row; for (i = 0; i < row_width; i++, sp += 3) { - if (*sp == trans_color->red && - *(sp + 1) == trans_color->green && - *(sp + 2) == trans_color->blue) + if (*sp == png_ptr->trans_color.red && + *(sp + 1) == png_ptr->trans_color.green && + *(sp + 2) == png_ptr->trans_color.blue) { - *sp = (png_byte)background->red; - *(sp + 1) = (png_byte)background->green; - *(sp + 2) = (png_byte)background->blue; + *sp = (png_byte)png_ptr->background.red; + *(sp + 1) = (png_byte)png_ptr->background.green; + *(sp + 2) = (png_byte)png_ptr->background.blue; } else @@ -3075,13 +3696,13 @@ png_do_background(png_row_infop row_info, png_bytep row, sp = row; for (i = 0; i < row_width; i++, sp += 3) { - if (*sp == trans_color->red && - *(sp + 1) == trans_color->green && - *(sp + 2) == trans_color->blue) + if (*sp == png_ptr->trans_color.red && + *(sp + 1) == png_ptr->trans_color.green && + *(sp + 2) == png_ptr->trans_color.blue) { - *sp = (png_byte)background->red; - *(sp + 1) = (png_byte)background->green; - *(sp + 2) = (png_byte)background->blue; + *sp = (png_byte)png_ptr->background.red; + *(sp + 1) = (png_byte)png_ptr->background.green; + *(sp + 2) = (png_byte)png_ptr->background.blue; } } } @@ -3102,16 +3723,17 @@ png_do_background(png_row_infop row_info, png_bytep row, png_uint_16 b = (png_uint_16)(((*(sp + 4)) << 8) + *(sp + 5)); - if (r == trans_color->red && g == trans_color->green && - b == trans_color->blue) + if (r == png_ptr->trans_color.red && + g == png_ptr->trans_color.green && + b == png_ptr->trans_color.blue) { /* Background is already in screen gamma */ - *sp = (png_byte)((background->red >> 8) & 0xff); - *(sp + 1) = (png_byte)(background->red & 0xff); - *(sp + 2) = (png_byte)((background->green >> 8) & 0xff); - *(sp + 3) = (png_byte)(background->green & 0xff); - *(sp + 4) = (png_byte)((background->blue >> 8) & 0xff); - *(sp + 5) = (png_byte)(background->blue & 0xff); + *sp = (png_byte)((png_ptr->background.red >> 8) & 0xff); + *(sp + 1) = (png_byte)(png_ptr->background.red & 0xff); + *(sp + 2) = (png_byte)((png_ptr->background.green >> 8) & 0xff); + *(sp + 3) = (png_byte)(png_ptr->background.green & 0xff); + *(sp + 4) = (png_byte)((png_ptr->background.blue >> 8) & 0xff); + *(sp + 5) = (png_byte)(png_ptr->background.blue & 0xff); } else @@ -3145,15 +3767,16 @@ png_do_background(png_row_infop row_info, png_bytep row, png_uint_16 b = (png_uint_16)(((*(sp + 4)) << 8) + *(sp + 5)); - if (r == trans_color->red && g == trans_color->green && - b == trans_color->blue) + if (r == png_ptr->trans_color.red && + g == png_ptr->trans_color.green && + b == png_ptr->trans_color.blue) { - *sp = (png_byte)((background->red >> 8) & 0xff); - *(sp + 1) = (png_byte)(background->red & 0xff); - *(sp + 2) = (png_byte)((background->green >> 8) & 0xff); - *(sp + 3) = (png_byte)(background->green & 0xff); - *(sp + 4) = (png_byte)((background->blue >> 8) & 0xff); - *(sp + 5) = (png_byte)(background->blue & 0xff); + *sp = (png_byte)((png_ptr->background.red >> 8) & 0xff); + *(sp + 1) = (png_byte)(png_ptr->background.red & 0xff); + *(sp + 2) = (png_byte)((png_ptr->background.green >> 8) & 0xff); + *(sp + 3) = (png_byte)(png_ptr->background.green & 0xff); + *(sp + 4) = (png_byte)((png_ptr->background.blue >> 8) & 0xff); + *(sp + 5) = (png_byte)(png_ptr->background.blue & 0xff); } } } @@ -3170,18 +3793,17 @@ png_do_background(png_row_infop row_info, png_bytep row, gamma_table != NULL) { sp = row; - dp = row; - for (i = 0; i < row_width; i++, sp += 2, dp++) + for (i = 0; i < row_width; i++, sp += 2) { png_uint_16 a = *(sp + 1); if (a == 0xff) - *dp = gamma_table[*sp]; + *sp = gamma_table[*sp]; else if (a == 0) { /* Background is already in screen gamma */ - *dp = (png_byte)background->gray; + *sp = (png_byte)png_ptr->background.gray; } else @@ -3189,8 +3811,10 @@ png_do_background(png_row_infop row_info, png_bytep row, png_byte v, w; v = gamma_to_1[*sp]; - png_composite(w, v, a, background_1->gray); - *dp = gamma_from_1[w]; + png_composite(w, v, a, png_ptr->background_1.gray); + if (!optimize) + w = gamma_from_1[w]; + *sp = w; } } } @@ -3198,24 +3822,15 @@ png_do_background(png_row_infop row_info, png_bytep row, #endif { sp = row; - dp = row; - for (i = 0; i < row_width; i++, sp += 2, dp++) + for (i = 0; i < row_width; i++, sp += 2) { png_byte a = *(sp + 1); - if (a == 0xff) - *dp = *sp; + if (a == 0) + *sp = (png_byte)png_ptr->background.gray; -#ifdef PNG_READ_GAMMA_SUPPORTED - else if (a == 0) - *dp = (png_byte)background->gray; - - else - png_composite(*dp, *sp, a, background_1->gray); - -#else - *dp = (png_byte)background->gray; -#endif + else if (a < 0xff) + png_composite(*sp, *sp, a, png_ptr->background_1.gray); } } } @@ -3226,8 +3841,7 @@ png_do_background(png_row_infop row_info, png_bytep row, gamma_16_to_1 != NULL) { sp = row; - dp = row; - for (i = 0; i < row_width; i++, sp += 4, dp += 2) + for (i = 0; i < row_width; i++, sp += 4) { png_uint_16 a = (png_uint_16)(((*(sp + 2)) << 8) + *(sp + 3)); @@ -3237,69 +3851,56 @@ png_do_background(png_row_infop row_info, png_bytep row, png_uint_16 v; v = gamma_16[*(sp + 1) >> gamma_shift][*sp]; - *dp = (png_byte)((v >> 8) & 0xff); - *(dp + 1) = (png_byte)(v & 0xff); + *sp = (png_byte)((v >> 8) & 0xff); + *(sp + 1) = (png_byte)(v & 0xff); } -#ifdef PNG_READ_GAMMA_SUPPORTED else if (a == 0) -#else - else -#endif { /* Background is already in screen gamma */ - *dp = (png_byte)((background->gray >> 8) & 0xff); - *(dp + 1) = (png_byte)(background->gray & 0xff); + *sp = (png_byte)((png_ptr->background.gray >> 8) & 0xff); + *(sp + 1) = (png_byte)(png_ptr->background.gray & 0xff); } -#ifdef PNG_READ_GAMMA_SUPPORTED else { png_uint_16 g, v, w; g = gamma_16_to_1[*(sp + 1) >> gamma_shift][*sp]; - png_composite_16(v, g, a, background_1->gray); - w = gamma_16_from_1[(v&0xff) >> gamma_shift][v >> 8]; - *dp = (png_byte)((w >> 8) & 0xff); - *(dp + 1) = (png_byte)(w & 0xff); + png_composite_16(v, g, a, png_ptr->background_1.gray); + if (optimize) + w = v; + else + w = gamma_16_from_1[(v&0xff) >> gamma_shift][v >> 8]; + *sp = (png_byte)((w >> 8) & 0xff); + *(sp + 1) = (png_byte)(w & 0xff); } -#endif } } else #endif { sp = row; - dp = row; - for (i = 0; i < row_width; i++, sp += 4, dp += 2) + for (i = 0; i < row_width; i++, sp += 4) { png_uint_16 a = (png_uint_16)(((*(sp + 2)) << 8) + *(sp + 3)); - if (a == (png_uint_16)0xffff) - png_memcpy(dp, sp, 2); - -#ifdef PNG_READ_GAMMA_SUPPORTED - else if (a == 0) -#else - else -#endif + if (a == 0) { - *dp = (png_byte)((background->gray >> 8) & 0xff); - *(dp + 1) = (png_byte)(background->gray & 0xff); + *sp = (png_byte)((png_ptr->background.gray >> 8) & 0xff); + *(sp + 1) = (png_byte)(png_ptr->background.gray & 0xff); } -#ifdef PNG_READ_GAMMA_SUPPORTED - else + else if (a < 0xffff) { png_uint_16 g, v; g = (png_uint_16)(((*sp) << 8) + *(sp + 1)); - png_composite_16(v, g, a, background_1->gray); - *dp = (png_byte)((v >> 8) & 0xff); - *(dp + 1) = (png_byte)(v & 0xff); + png_composite_16(v, g, a, png_ptr->background_1.gray); + *sp = (png_byte)((v >> 8) & 0xff); + *(sp + 1) = (png_byte)(v & 0xff); } -#endif } } } @@ -3315,24 +3916,23 @@ png_do_background(png_row_infop row_info, png_bytep row, gamma_table != NULL) { sp = row; - dp = row; - for (i = 0; i < row_width; i++, sp += 4, dp += 3) + for (i = 0; i < row_width; i++, sp += 4) { png_byte a = *(sp + 3); if (a == 0xff) { - *dp = gamma_table[*sp]; - *(dp + 1) = gamma_table[*(sp + 1)]; - *(dp + 2) = gamma_table[*(sp + 2)]; + *sp = gamma_table[*sp]; + *(sp + 1) = gamma_table[*(sp + 1)]; + *(sp + 2) = gamma_table[*(sp + 2)]; } else if (a == 0) { /* Background is already in screen gamma */ - *dp = (png_byte)background->red; - *(dp + 1) = (png_byte)background->green; - *(dp + 2) = (png_byte)background->blue; + *sp = (png_byte)png_ptr->background.red; + *(sp + 1) = (png_byte)png_ptr->background.green; + *(sp + 2) = (png_byte)png_ptr->background.blue; } else @@ -3340,16 +3940,19 @@ png_do_background(png_row_infop row_info, png_bytep row, png_byte v, w; v = gamma_to_1[*sp]; - png_composite(w, v, a, background_1->red); - *dp = gamma_from_1[w]; + png_composite(w, v, a, png_ptr->background_1.red); + if (!optimize) w = gamma_from_1[w]; + *sp = w; v = gamma_to_1[*(sp + 1)]; - png_composite(w, v, a, background_1->green); - *(dp + 1) = gamma_from_1[w]; + png_composite(w, v, a, png_ptr->background_1.green); + if (!optimize) w = gamma_from_1[w]; + *(sp + 1) = w; v = gamma_to_1[*(sp + 2)]; - png_composite(w, v, a, background_1->blue); - *(dp + 2) = gamma_from_1[w]; + png_composite(w, v, a, png_ptr->background_1.blue); + if (!optimize) w = gamma_from_1[w]; + *(sp + 2) = w; } } } @@ -3357,34 +3960,26 @@ png_do_background(png_row_infop row_info, png_bytep row, #endif { sp = row; - dp = row; - for (i = 0; i < row_width; i++, sp += 4, dp += 3) + for (i = 0; i < row_width; i++, sp += 4) { png_byte a = *(sp + 3); - if (a == 0xff) + if (a == 0) { - *dp = *sp; - *(dp + 1) = *(sp + 1); - *(dp + 2) = *(sp + 2); + *sp = (png_byte)png_ptr->background.red; + *(sp + 1) = (png_byte)png_ptr->background.green; + *(sp + 2) = (png_byte)png_ptr->background.blue; } - else if (a == 0) + else if (a < 0xff) { - *dp = (png_byte)background->red; - *(dp + 1) = (png_byte)background->green; - *(dp + 2) = (png_byte)background->blue; - } + png_composite(*sp, *sp, a, png_ptr->background.red); - else - { - png_composite(*dp, *sp, a, background->red); + png_composite(*(sp + 1), *(sp + 1), a, + png_ptr->background.green); - png_composite(*(dp + 1), *(sp + 1), a, - background->green); - - png_composite(*(dp + 2), *(sp + 2), a, - background->blue); + png_composite(*(sp + 2), *(sp + 2), a, + png_ptr->background.blue); } } } @@ -3396,8 +3991,7 @@ png_do_background(png_row_infop row_info, png_bytep row, gamma_16_to_1 != NULL) { sp = row; - dp = row; - for (i = 0; i < row_width; i++, sp += 8, dp += 6) + for (i = 0; i < row_width; i++, sp += 8) { png_uint_16 a = (png_uint_16)(((png_uint_16)(*(sp + 6)) << 8) + (png_uint_16)(*(sp + 7))); @@ -3407,53 +4001,55 @@ png_do_background(png_row_infop row_info, png_bytep row, png_uint_16 v; v = gamma_16[*(sp + 1) >> gamma_shift][*sp]; - *dp = (png_byte)((v >> 8) & 0xff); - *(dp + 1) = (png_byte)(v & 0xff); + *sp = (png_byte)((v >> 8) & 0xff); + *(sp + 1) = (png_byte)(v & 0xff); v = gamma_16[*(sp + 3) >> gamma_shift][*(sp + 2)]; - *(dp + 2) = (png_byte)((v >> 8) & 0xff); - *(dp + 3) = (png_byte)(v & 0xff); + *(sp + 2) = (png_byte)((v >> 8) & 0xff); + *(sp + 3) = (png_byte)(v & 0xff); v = gamma_16[*(sp + 5) >> gamma_shift][*(sp + 4)]; - *(dp + 4) = (png_byte)((v >> 8) & 0xff); - *(dp + 5) = (png_byte)(v & 0xff); + *(sp + 4) = (png_byte)((v >> 8) & 0xff); + *(sp + 5) = (png_byte)(v & 0xff); } else if (a == 0) { /* Background is already in screen gamma */ - *dp = (png_byte)((background->red >> 8) & 0xff); - *(dp + 1) = (png_byte)(background->red & 0xff); - *(dp + 2) = (png_byte)((background->green >> 8) & 0xff); - *(dp + 3) = (png_byte)(background->green & 0xff); - *(dp + 4) = (png_byte)((background->blue >> 8) & 0xff); - *(dp + 5) = (png_byte)(background->blue & 0xff); + *sp = (png_byte)((png_ptr->background.red >> 8) & 0xff); + *(sp + 1) = (png_byte)(png_ptr->background.red & 0xff); + *(sp + 2) = (png_byte)((png_ptr->background.green >> 8) & 0xff); + *(sp + 3) = (png_byte)(png_ptr->background.green & 0xff); + *(sp + 4) = (png_byte)((png_ptr->background.blue >> 8) & 0xff); + *(sp + 5) = (png_byte)(png_ptr->background.blue & 0xff); } else { - png_uint_16 v, w, x; + png_uint_16 v, w; v = gamma_16_to_1[*(sp + 1) >> gamma_shift][*sp]; - png_composite_16(w, v, a, background_1->red); - - x = gamma_16_from_1[((w&0xff) >> gamma_shift)][w >> 8]; - *dp = (png_byte)((x >> 8) & 0xff); - *(dp + 1) = (png_byte)(x & 0xff); + png_composite_16(w, v, a, png_ptr->background_1.red); + if (!optimize) + w = gamma_16_from_1[((w&0xff) >> gamma_shift)][w >> 8]; + *sp = (png_byte)((w >> 8) & 0xff); + *(sp + 1) = (png_byte)(w & 0xff); v = gamma_16_to_1[*(sp + 3) >> gamma_shift][*(sp + 2)]; - png_composite_16(w, v, a, background_1->green); + png_composite_16(w, v, a, png_ptr->background_1.green); + if (!optimize) + w = gamma_16_from_1[((w&0xff) >> gamma_shift)][w >> 8]; - x = gamma_16_from_1[((w&0xff) >> gamma_shift)][w >> 8]; - *(dp + 2) = (png_byte)((x >> 8) & 0xff); - *(dp + 3) = (png_byte)(x & 0xff); + *(sp + 2) = (png_byte)((w >> 8) & 0xff); + *(sp + 3) = (png_byte)(w & 0xff); v = gamma_16_to_1[*(sp + 5) >> gamma_shift][*(sp + 4)]; - png_composite_16(w, v, a, background_1->blue); + png_composite_16(w, v, a, png_ptr->background_1.blue); + if (!optimize) + w = gamma_16_from_1[((w&0xff) >> gamma_shift)][w >> 8]; - x = gamma_16_from_1[(w & 0xff) >> gamma_shift][w >> 8]; - *(dp + 4) = (png_byte)((x >> 8) & 0xff); - *(dp + 5) = (png_byte)(x & 0xff); + *(sp + 4) = (png_byte)((w >> 8) & 0xff); + *(sp + 5) = (png_byte)(w & 0xff); } } } @@ -3462,28 +4058,22 @@ png_do_background(png_row_infop row_info, png_bytep row, #endif { sp = row; - dp = row; - for (i = 0; i < row_width; i++, sp += 8, dp += 6) + for (i = 0; i < row_width; i++, sp += 8) { png_uint_16 a = (png_uint_16)(((png_uint_16)(*(sp + 6)) << 8) + (png_uint_16)(*(sp + 7))); - if (a == (png_uint_16)0xffff) + if (a == 0) { - png_memcpy(dp, sp, 6); + *sp = (png_byte)((png_ptr->background.red >> 8) & 0xff); + *(sp + 1) = (png_byte)(png_ptr->background.red & 0xff); + *(sp + 2) = (png_byte)((png_ptr->background.green >> 8) & 0xff); + *(sp + 3) = (png_byte)(png_ptr->background.green & 0xff); + *(sp + 4) = (png_byte)((png_ptr->background.blue >> 8) & 0xff); + *(sp + 5) = (png_byte)(png_ptr->background.blue & 0xff); } - else if (a == 0) - { - *dp = (png_byte)((background->red >> 8) & 0xff); - *(dp + 1) = (png_byte)(background->red & 0xff); - *(dp + 2) = (png_byte)((background->green >> 8) & 0xff); - *(dp + 3) = (png_byte)(background->green & 0xff); - *(dp + 4) = (png_byte)((background->blue >> 8) & 0xff); - *(dp + 5) = (png_byte)(background->blue & 0xff); - } - - else + else if (a < 0xffff) { png_uint_16 v; @@ -3493,17 +4083,17 @@ png_do_background(png_row_infop row_info, png_bytep row, png_uint_16 b = (png_uint_16)(((*(sp + 4)) << 8) + *(sp + 5)); - png_composite_16(v, r, a, background->red); - *dp = (png_byte)((v >> 8) & 0xff); - *(dp + 1) = (png_byte)(v & 0xff); + png_composite_16(v, r, a, png_ptr->background.red); + *sp = (png_byte)((v >> 8) & 0xff); + *(sp + 1) = (png_byte)(v & 0xff); - png_composite_16(v, g, a, background->green); - *(dp + 2) = (png_byte)((v >> 8) & 0xff); - *(dp + 3) = (png_byte)(v & 0xff); + png_composite_16(v, g, a, png_ptr->background.green); + *(sp + 2) = (png_byte)((v >> 8) & 0xff); + *(sp + 3) = (png_byte)(v & 0xff); - png_composite_16(v, b, a, background->blue); - *(dp + 4) = (png_byte)((v >> 8) & 0xff); - *(dp + 5) = (png_byte)(v & 0xff); + png_composite_16(v, b, a, png_ptr->background.blue); + *(sp + 4) = (png_byte)((v >> 8) & 0xff); + *(sp + 5) = (png_byte)(v & 0xff); } } } @@ -3514,16 +4104,6 @@ png_do_background(png_row_infop row_info, png_bytep row, default: break; } - - if (row_info->color_type & PNG_COLOR_MASK_ALPHA) - { - row_info->color_type = (png_byte)(row_info->color_type & - ~PNG_COLOR_MASK_ALPHA); - row_info->channels--; - row_info->pixel_depth = (png_byte)(row_info->channels * - row_info->bit_depth); - row_info->rowbytes = PNG_ROWBYTES(row_info->pixel_depth, row_width); - } } } #endif @@ -3536,10 +4116,12 @@ png_do_background(png_row_infop row_info, png_bytep row, * build_gamma_table(). */ void /* PRIVATE */ -png_do_gamma(png_row_infop row_info, png_bytep row, - png_const_bytep gamma_table, png_const_uint_16pp gamma_16_table, - int gamma_shift) +png_do_gamma(png_row_infop row_info, png_bytep row, png_structp png_ptr) { + png_const_bytep gamma_table = png_ptr->gamma_table; + png_const_uint_16pp gamma_16_table = png_ptr->gamma_16_table; + int gamma_shift = png_ptr->gamma_shift; + png_bytep sp; png_uint_32 i; png_uint_32 row_width=row_info->width; @@ -3729,6 +4311,73 @@ png_do_gamma(png_row_infop row_info, png_bytep row, } #endif +#ifdef PNG_READ_ALPHA_MODE_SUPPORTED +/* Encode the alpha channel to the output gamma (the input channel is always + * linear.) Called only with color types that have an alpha channel. Needs the + * from_1 tables. + */ +void /* PRIVATE */ +png_do_encode_alpha(png_row_infop row_info, png_bytep row, png_structp png_ptr) +{ + png_uint_32 row_width = row_info->width; + + png_debug(1, "in png_do_encode_alpha"); + + if (row_info->color_type & PNG_COLOR_MASK_ALPHA) + { + if (row_info->bit_depth == 8) + { + PNG_CONST png_bytep table = png_ptr->gamma_from_1; + + if (table != NULL) + { + PNG_CONST int step = + (row_info->color_type & PNG_COLOR_MASK_COLOR) ? 4 : 2; + + /* The alpha channel is the last component: */ + row += step - 1; + + for (; row_width > 0; --row_width, row += step) + *row = table[*row]; + + return; + } + } + + else if (row_info->bit_depth == 16) + { + PNG_CONST png_uint_16pp table = png_ptr->gamma_16_from_1; + PNG_CONST int gamma_shift = png_ptr->gamma_shift; + + if (table != NULL) + { + PNG_CONST int step = + (row_info->color_type & PNG_COLOR_MASK_COLOR) ? 8 : 4; + + /* The alpha channel is the last component: */ + row += step - 2; + + for (; row_width > 0; --row_width, row += step) + { + png_uint_16 v; + + v = table[*(row + 1) >> gamma_shift][*row]; + *row = (png_byte)((v >> 8) & 0xff); + *(row + 1) = (png_byte)(v & 0xff); + } + + return; + } + } + } + + /* Only get to here if called with a weird row_info; no harm has been done, + * so just issue a warning. + */ + png_warning(png_ptr, "png_do_encode_alpha: unexpected call"); +} +#endif + #ifdef PNG_READ_EXPAND_SUPPORTED /* Expands a palette row to an RGB or RGBA row depending * upon whether you supply trans and num_trans. @@ -3834,7 +4483,7 @@ png_do_expand_palette(png_row_infop row_info, png_bytep row, if (row_info->bit_depth == 8) { { - if (trans_alpha != NULL) + if (num_trans > 0) { sp = row + (png_size_t)row_width - 1; dp = row + (png_size_t)(row_width << 2) - 1; @@ -3888,7 +4537,7 @@ png_do_expand_palette(png_row_infop row_info, png_bytep row, */ void /* PRIVATE */ png_do_expand(png_row_infop row_info, png_bytep row, - png_const_color_16p trans_value) + png_const_color_16p trans_color) { int shift, value; png_bytep sp, dp; @@ -3900,7 +4549,7 @@ png_do_expand(png_row_infop row_info, png_bytep row, { if (row_info->color_type == PNG_COLOR_TYPE_GRAY) { - png_uint_16 gray = (png_uint_16)(trans_value ? trans_value->gray : 0); + png_uint_16 gray = (png_uint_16)(trans_color ? trans_color->gray : 0); if (row_info->bit_depth < 8) { @@ -3992,7 +4641,7 @@ png_do_expand(png_row_infop row_info, png_bytep row, row_info->rowbytes = row_width; } - if (trans_value != NULL) + if (trans_color != NULL) { if (row_info->bit_depth == 8) { @@ -4044,13 +4693,13 @@ png_do_expand(png_row_infop row_info, png_bytep row, row_width); } } - else if (row_info->color_type == PNG_COLOR_TYPE_RGB && trans_value) + else if (row_info->color_type == PNG_COLOR_TYPE_RGB && trans_color) { if (row_info->bit_depth == 8) { - png_byte red = (png_byte)(trans_value->red & 0xff); - png_byte green = (png_byte)(trans_value->green & 0xff); - png_byte blue = (png_byte)(trans_value->blue & 0xff); + png_byte red = (png_byte)(trans_color->red & 0xff); + png_byte green = (png_byte)(trans_color->green & 0xff); + png_byte blue = (png_byte)(trans_color->blue & 0xff); sp = row + (png_size_t)row_info->rowbytes - 1; dp = row + (png_size_t)(row_width << 2) - 1; for (i = 0; i < row_width; i++) @@ -4068,12 +4717,12 @@ png_do_expand(png_row_infop row_info, png_bytep row, } else if (row_info->bit_depth == 16) { - png_byte red_high = (png_byte)((trans_value->red >> 8) & 0xff); - png_byte green_high = (png_byte)((trans_value->green >> 8) & 0xff); - png_byte blue_high = (png_byte)((trans_value->blue >> 8) & 0xff); - png_byte red_low = (png_byte)(trans_value->red & 0xff); - png_byte green_low = (png_byte)(trans_value->green & 0xff); - png_byte blue_low = (png_byte)(trans_value->blue & 0xff); + png_byte red_high = (png_byte)((trans_color->red >> 8) & 0xff); + png_byte green_high = (png_byte)((trans_color->green >> 8) & 0xff); + png_byte blue_high = (png_byte)((trans_color->blue >> 8) & 0xff); + png_byte red_low = (png_byte)(trans_color->red & 0xff); + png_byte green_low = (png_byte)(trans_color->green & 0xff); + png_byte blue_low = (png_byte)(trans_color->blue & 0xff); sp = row + row_info->rowbytes - 1; dp = row + (png_size_t)(row_width << 3) - 1; for (i = 0; i < row_width; i++) @@ -4236,6 +4885,7 @@ png_do_quantize(png_row_infop row_info, png_bytep row, } } #endif /* PNG_READ_QUANTIZE_SUPPORTED */ +#endif /* PNG_READ_TRANSFORMS_SUPPORTED */ #ifdef PNG_MNG_FEATURES_SUPPORTED /* Undoes intrapixel differencing */ diff --git a/harbour/src/3rd/png/pngrutil.c b/harbour/src/3rd/png/pngrutil.c index d476431744..07e46e2fed 100644 --- a/harbour/src/3rd/png/pngrutil.c +++ b/harbour/src/3rd/png/pngrutil.c @@ -1,7 +1,7 @@ /* pngrutil.c - utilities to read a PNG file * - * Last changed in libpng 1.5.2 [March 31, 2011] + * Last changed in libpng 1.5.4 [July 7, 2011] * Copyright (c) 1998-2011 Glenn Randers-Pehrson * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) @@ -278,8 +278,7 @@ png_crc_error(png_structp png_ptr) return (0); } -#if defined(PNG_READ_zTXt_SUPPORTED) || defined(PNG_READ_iTXt_SUPPORTED) || \ - defined(PNG_READ_iCCP_SUPPORTED) +#ifdef PNG_READ_COMPRESSED_TEXT_SUPPORTED static png_size_t png_inflate(png_structp png_ptr, png_bytep data, png_size_t size, png_bytep output, png_size_t output_size) @@ -370,41 +369,31 @@ png_inflate(png_structp png_ptr, png_bytep data, png_size_t size, * and the error message is dumped into the uncompressed * buffer if available. */ +# ifdef PNG_WARNINGS_SUPPORTED { - PNG_CONST char *msg; -#ifdef PNG_CONSOLE_IO_SUPPORTED - char umsg[52]; -#endif + png_const_charp msg; + if (png_ptr->zstream.msg != 0) msg = png_ptr->zstream.msg; - else + else switch (ret) { -#ifdef PNG_CONSOLE_IO_SUPPORTED - switch (ret) - { - case Z_BUF_ERROR: - msg = "Buffer error in compressed datastream in %s chunk"; - break; + case Z_BUF_ERROR: + msg = "Buffer error in compressed datastream"; + break; - case Z_DATA_ERROR: - msg = "Data error in compressed datastream in %s chunk"; - break; + case Z_DATA_ERROR: + msg = "Data error in compressed datastream"; + break; - default: - msg = "Incomplete compressed datastream in %s chunk"; - break; - } - - png_snprintf(umsg, sizeof umsg, msg, png_ptr->chunk_name); - msg = umsg; -#else - msg = "Damaged compressed datastream in chunk other than IDAT"; -#endif + default: + msg = "Incomplete compressed datastream"; + break; } - png_warning(png_ptr, msg); + png_chunk_warning(png_ptr, msg); } +# endif /* 0 means an error - notice that this code simply ignores * zero length compressed chunks as a result. @@ -438,8 +427,8 @@ png_decompress_chunk(png_structp png_ptr, int comp_type, png_size_t expanded_size = png_inflate(png_ptr, (png_bytep)(png_ptr->chunkdata + prefix_size), chunklength - prefix_size, - 0, /*output*/ - 0); /*output size*/ + 0, /* output */ + 0); /* output size */ /* Now check the limits on this chunk - if the limit fails the * compressed data will be removed, the prefix will remain. @@ -500,15 +489,9 @@ png_decompress_chunk(png_structp png_ptr, int comp_type, else /* if (comp_type != PNG_COMPRESSION_TYPE_BASE) */ { -#ifdef PNG_STDIO_SUPPORTED - char umsg[50]; - - png_snprintf(umsg, sizeof umsg, - "Unknown zTXt compression type %d", comp_type); - png_warning(png_ptr, umsg); -#else - png_warning(png_ptr, "Unknown zTXt compression type"); -#endif + PNG_WARNING_PARAMETERS(p) + png_warning_parameter_signed(p, 1, PNG_NUMBER_FORMAT_d, comp_type); + png_formatted_warning(png_ptr, p, "Unknown zTXt compression type @1"); /* The recovery is to simply drop the data. */ } @@ -536,7 +519,7 @@ png_decompress_chunk(png_structp png_ptr, int comp_type, *newlength = prefix_size; } -#endif +#endif /* PNG_READ_COMPRESSED_TEXT_SUPPORTED */ /* Read and check the IDHR chunk */ void /* PRIVATE */ @@ -846,12 +829,10 @@ png_handle_gAMA(png_structp png_ptr, png_infop info_ptr, png_uint_32 length) { if (PNG_OUT_OF_RANGE(igamma, 45500L, 500)) { - png_warning(png_ptr, - "Ignoring incorrect gAMA value when sRGB is also present"); - -# ifdef PNG_CONSOLE_IO_SUPPORTED - fprintf(stderr, "gamma = (%d/100000)", (int)igamma); -# endif + PNG_WARNING_PARAMETERS(p) + png_warning_parameter_signed(p, 1, PNG_NUMBER_FORMAT_fixed, igamma); + png_formatted_warning(png_ptr, p, + "Ignoring incorrect gAMA value @1 when sRGB is also present"); return; } } @@ -1020,21 +1001,52 @@ png_handle_cHRM(png_structp png_ptr, png_infop info_ptr, png_uint_32 length) PNG_OUT_OF_RANGE(x_blue, 15000, 1000) || PNG_OUT_OF_RANGE(y_blue, 6000, 1000)) { - png_warning(png_ptr, - "Ignoring incorrect cHRM value when sRGB is also present"); + PNG_WARNING_PARAMETERS(p) -#ifdef PNG_CONSOLE_IO_SUPPORTED - fprintf(stderr, "wx=%d, wy=%d, rx=%d, ry=%d\n", - x_white, y_white, x_red, y_red); + png_warning_parameter_signed(p, 1, PNG_NUMBER_FORMAT_fixed, x_white); + png_warning_parameter_signed(p, 2, PNG_NUMBER_FORMAT_fixed, y_white); + png_warning_parameter_signed(p, 3, PNG_NUMBER_FORMAT_fixed, x_red); + png_warning_parameter_signed(p, 4, PNG_NUMBER_FORMAT_fixed, y_red); + png_warning_parameter_signed(p, 5, PNG_NUMBER_FORMAT_fixed, x_green); + png_warning_parameter_signed(p, 6, PNG_NUMBER_FORMAT_fixed, y_green); + png_warning_parameter_signed(p, 7, PNG_NUMBER_FORMAT_fixed, x_blue); + png_warning_parameter_signed(p, 8, PNG_NUMBER_FORMAT_fixed, y_blue); - fprintf(stderr, "gx=%d, gy=%d, bx=%d, by=%d\n", - x_green, y_green, x_blue, y_blue); -#endif /* PNG_CONSOLE_IO_SUPPORTED */ + png_formatted_warning(png_ptr, p, + "Ignoring incorrect cHRM white(@1,@2) r(@3,@4)g(@5,@6)b(@7,@8) " + "when sRGB is also present"); } return; } #endif /* PNG_READ_sRGB_SUPPORTED */ +#ifdef PNG_READ_RGB_TO_GRAY_SUPPORTED + /* Store the _white values as default coefficients for the rgb to gray + * operation if it is supported. + */ + if ((png_ptr->transformations & PNG_RGB_TO_GRAY) == 0) + { + /* png_set_background has not been called, the coefficients must be in + * range for the following to work without overflow. + */ + if (y_red <= (1<<17) && y_green <= (1<<17) && y_blue <= (1<<17)) + { + /* The y values are chromaticities: Y/X+Y+Z, the weights for the gray + * transformation are simply the normalized Y values for red, green and + * blue scaled by 32768. + */ + png_uint_32 w = y_red + y_green + y_blue; + + png_ptr->rgb_to_gray_red_coeff = (png_uint_16)(((png_uint_32)y_red * + 32768)/w); + png_ptr->rgb_to_gray_green_coeff = (png_uint_16)(((png_uint_32)y_green + * 32768)/w); + png_ptr->rgb_to_gray_blue_coeff = (png_uint_16)(((png_uint_32)y_blue * + 32768)/w); + } + } +#endif + png_set_cHRM_fixed(png_ptr, info_ptr, x_white, y_white, x_red, y_red, x_green, y_green, x_blue, y_blue); } @@ -1096,11 +1108,13 @@ png_handle_sRGB(png_structp png_ptr, png_infop info_ptr, png_uint_32 length) { if (PNG_OUT_OF_RANGE(info_ptr->gamma, 45500L, 500)) { - png_warning(png_ptr, - "Ignoring incorrect gAMA value when sRGB is also present"); -#ifdef PNG_CONSOLE_IO_SUPPORTED - fprintf(stderr, "incorrect gamma=(%d/100000)\n", info_ptr->gamma); -#endif + PNG_WARNING_PARAMETERS(p) + + png_warning_parameter_signed(p, 1, PNG_NUMBER_FORMAT_fixed, + info_ptr->gamma); + + png_formatted_warning(png_ptr, p, + "Ignoring incorrect gAMA value @1 when sRGB is also present"); } } #endif /* PNG_READ_gAMA_SUPPORTED */ @@ -1240,23 +1254,15 @@ png_handle_iCCP(png_structp png_ptr, png_infop info_ptr, png_uint_32 length) /* And the following guarantees that profile_size == profile_length. */ if (profile_size > profile_length) { + PNG_WARNING_PARAMETERS(p) + png_free(png_ptr, png_ptr->chunkdata); png_ptr->chunkdata = NULL; -#ifdef PNG_STDIO_SUPPORTED - { - char umsg[80]; - png_snprintf2(umsg, 80, - "Ignoring iCCP chunk with declared size = %u " - "and actual length = %u", - (unsigned int) profile_size, - (unsigned int) profile_length); - png_warning(png_ptr, umsg); - } -#else - png_warning(png_ptr, - "Ignoring iCCP chunk with uncompressed size mismatch"); -#endif + png_warning_parameter_unsigned(p, 1, PNG_NUMBER_FORMAT_u, profile_size); + png_warning_parameter_unsigned(p, 2, PNG_NUMBER_FORMAT_u, profile_length); + png_formatted_warning(png_ptr, p, + "Ignoring iCCP chunk with declared size = @1 and actual length = @2"); return; } @@ -1565,6 +1571,7 @@ png_handle_bKGD(png_structp png_ptr, png_infop info_ptr, png_uint_32 length) { png_size_t truelen; png_byte buf[6]; + png_color_16 background; png_debug(1, "in png_handle_bKGD"); @@ -1621,7 +1628,7 @@ png_handle_bKGD(png_structp png_ptr, png_infop info_ptr, png_uint_32 length) */ if (png_ptr->color_type == PNG_COLOR_TYPE_PALETTE) { - png_ptr->background.index = buf[0]; + background.index = buf[0]; if (info_ptr && info_ptr->num_palette) { @@ -1631,33 +1638,36 @@ png_handle_bKGD(png_structp png_ptr, png_infop info_ptr, png_uint_32 length) return; } - png_ptr->background.red = - (png_uint_16)png_ptr->palette[buf[0]].red; - - png_ptr->background.green = - (png_uint_16)png_ptr->palette[buf[0]].green; - - png_ptr->background.blue = - (png_uint_16)png_ptr->palette[buf[0]].blue; + background.red = (png_uint_16)png_ptr->palette[buf[0]].red; + background.green = (png_uint_16)png_ptr->palette[buf[0]].green; + background.blue = (png_uint_16)png_ptr->palette[buf[0]].blue; } + + else + background.red = background.green = background.blue = 0; + + background.gray = 0; } else if (!(png_ptr->color_type & PNG_COLOR_MASK_COLOR)) /* GRAY */ { - png_ptr->background.red = - png_ptr->background.green = - png_ptr->background.blue = - png_ptr->background.gray = png_get_uint_16(buf); + background.index = 0; + background.red = + background.green = + background.blue = + background.gray = png_get_uint_16(buf); } else { - png_ptr->background.red = png_get_uint_16(buf); - png_ptr->background.green = png_get_uint_16(buf + 2); - png_ptr->background.blue = png_get_uint_16(buf + 4); + background.index = 0; + background.red = png_get_uint_16(buf); + background.green = png_get_uint_16(buf + 2); + background.blue = png_get_uint_16(buf + 4); + background.gray = 0; } - png_set_bKGD(png_ptr, info_ptr, &(png_ptr->background)); + png_set_bKGD(png_ptr, info_ptr, &background); } #endif @@ -1982,6 +1992,14 @@ png_handle_sCAL(png_structp png_ptr, png_infop info_ptr, png_uint_32 length) return; } + /* Need unit type, width, \0, height: minimum 4 bytes */ + else if (length < 4) + { + png_warning(png_ptr, "sCAL chunk too short"); + png_crc_finish(png_ptr, length); + return; + } + png_debug1(2, "Allocating and reading sCAL chunk data (%u bytes)", length + 1); @@ -2017,23 +2035,29 @@ png_handle_sCAL(png_structp png_ptr, png_infop info_ptr, png_uint_32 length) /* Validate the ASCII numbers, need two ASCII numbers separated by * a '\0' and they need to fit exactly in the chunk data. */ - i = 0; + i = 1; state = 0; - if (png_ptr->chunkdata[1] == 45 /* negative width */ || - !png_check_fp_number(png_ptr->chunkdata, slength, &state, &i) || + if (!png_check_fp_number(png_ptr->chunkdata, slength, &state, &i) || i >= slength || png_ptr->chunkdata[i++] != 0) png_warning(png_ptr, "Invalid sCAL chunk ignored: bad width format"); + else if (!PNG_FP_IS_POSITIVE(state)) + png_warning(png_ptr, "Invalid sCAL chunk ignored: non-positive width"); + else { png_size_t heighti = i; - if (png_ptr->chunkdata[i] == 45 /* negative height */ || - !png_check_fp_number(png_ptr->chunkdata, slength, &state, &i) || + state = 0; + if (!png_check_fp_number(png_ptr->chunkdata, slength, &state, &i) || i != slength) png_warning(png_ptr, "Invalid sCAL chunk ignored: bad height format"); + else if (!PNG_FP_IS_POSITIVE(state)) + png_warning(png_ptr, + "Invalid sCAL chunk ignored: non-positive height"); + else /* This is the (only) success case. */ png_set_sCAL_s(png_ptr, info_ptr, png_ptr->chunkdata[0], @@ -2660,6 +2684,14 @@ png_combine_row(png_structp png_ptr, png_bytep row, int mask) { png_debug(1, "in png_combine_row"); + /* Added in 1.5.4: the row_info should match the information returned by any + * call to png_read_update_info at this point. Do not continue if we got + * this wrong. + */ + if (png_ptr->info_rowbytes != 0 && png_ptr->info_rowbytes != + PNG_ROWBYTES(png_ptr->row_info.pixel_depth, png_ptr->width)) + png_error(png_ptr, "internal row size calculation error"); + if (mask == 0xff) { png_memcpy(row, png_ptr->row_buf + 1, @@ -3398,7 +3430,9 @@ png_read_start_row(png_structp png_ptr) png_debug(1, "in png_read_start_row"); png_ptr->zstream.avail_in = 0; +#ifdef PNG_READ_TRANSFORMS_SUPPORTED png_init_read_transformations(png_ptr); +#endif #ifdef PNG_READ_INTERLACING_SUPPORTED if (png_ptr->interlaced) { diff --git a/harbour/src/3rd/png/pngset.c b/harbour/src/3rd/png/pngset.c index 2cfcf33bdf..7eaad7dd45 100644 --- a/harbour/src/3rd/png/pngset.c +++ b/harbour/src/3rd/png/pngset.c @@ -1,7 +1,7 @@ /* pngset.c - storage of image information into info struct * - * Last changed in libpng 1.5.1 [February 3, 2011] + * Last changed in libpng 1.5.4 [July 7, 2011] * Copyright (c) 1998-2011 Glenn Randers-Pehrson * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) @@ -94,15 +94,16 @@ png_set_gAMA_fixed(png_structp png_ptr, png_infop info_ptr, png_fixed_point if (png_ptr == NULL || info_ptr == NULL) return; - /* Previously these values were limited, however they must be - * wrong, therefore storing them (and setting PNG_INFO_gAMA) - * must be wrong too. + /* Changed in libpng-1.5.4 to limit the values to ensure overflow can't + * occur. Since the fixed point representation is assymetrical it is + * possible for 1/gamma to overflow the limit of 21474 and this means the + * gamma value must be at least 5/100000 and hence at most 20000.0. For + * safety the limits here are a little narrower. The values are 0.00016 to + * 6250.0, which are truely ridiculous gammma values (and will produce + * displays that are all black or all white.) */ - if (file_gamma > (png_fixed_point)PNG_UINT_31_MAX) - png_warning(png_ptr, "Gamma too large, ignored"); - - else if (file_gamma <= 0) - png_warning(png_ptr, "Negative or zero gamma ignored"); + if (file_gamma < 16 || file_gamma > 625000000) + png_warning(png_ptr, "Out of range gamma value ignored"); else { @@ -340,12 +341,12 @@ png_set_sCAL_s(png_structp png_ptr, png_infop info_ptr, if (unit != 1 && unit != 2) png_error(png_ptr, "Invalid sCAL unit"); - if (swidth == NULL || (lengthw = png_strlen(swidth)) <= 0 || - swidth[0] == 45 /*'-'*/ || !png_check_fp_string(swidth, lengthw)) + if (swidth == NULL || (lengthw = png_strlen(swidth)) == 0 || + swidth[0] == 45 /* '-' */ || !png_check_fp_string(swidth, lengthw)) png_error(png_ptr, "Invalid sCAL width"); - if (sheight == NULL || (lengthh = png_strlen(sheight)) <= 0 || - sheight[0] == 45 /*'-'*/ || !png_check_fp_string(sheight, lengthh)) + if (sheight == NULL || (lengthh = png_strlen(sheight)) == 0 || + sheight[0] == 45 /* '-' */ || !png_check_fp_string(sheight, lengthh)) png_error(png_ptr, "Invalid sCAL height"); info_ptr->scal_unit = (png_byte)unit; @@ -545,7 +546,7 @@ png_set_sRGB_gAMA_and_cHRM(png_structp png_ptr, png_infop info_ptr, png_set_sRGB(png_ptr, info_ptr, srgb_intent); # ifdef PNG_gAMA_SUPPORTED - png_set_gAMA_fixed(png_ptr, info_ptr, 45455L); + png_set_gAMA_fixed(png_ptr, info_ptr, PNG_GAMMA_sRGB_INVERSE); # endif # ifdef PNG_cHRM_SUPPORTED diff --git a/harbour/src/3rd/png/pngstru.h b/harbour/src/3rd/png/pngstru.h index 8d781faefe..93b2b3eb0f 100644 --- a/harbour/src/3rd/png/pngstru.h +++ b/harbour/src/3rd/png/pngstru.h @@ -5,7 +5,7 @@ * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) * - * Last changed in libpng 1.5.0 [January 6, 2011] + * Last changed in libpng 1.5.4 [July 7, 2011] * * This code is released under the libpng license. * For conditions of distribution and use, see the disclaimer @@ -29,11 +29,13 @@ struct png_struct_def { #ifdef PNG_SETJMP_SUPPORTED - jmp_buf png_jmpbuf; /* used in png_error */ + jmp_buf longjmp_buffer; /* used in png_error */ png_longjmp_ptr longjmp_fn;/* setjmp non-local goto function. */ #endif png_error_ptr error_fn; /* function for printing errors and aborting */ +#ifdef PNG_WARNINGS_SUPPORTED png_error_ptr warning_fn; /* function for printing warnings */ +#endif png_voidp error_ptr; /* user supplied struct for error functions */ png_rw_ptr write_data_fn; /* function for writing output data */ png_rw_ptr read_data_fn; /* function for reading input data */ @@ -64,11 +66,36 @@ struct png_struct_def z_stream zstream; /* pointer to decompression structure (below) */ png_bytep zbuf; /* buffer for zlib */ uInt zbuf_size; /* size of zbuf (typically 65536) */ +#ifdef PNG_WRITE_SUPPORTED + +/* Added in 1.5.4: state to keep track of whether the zstream has been + * initialized and if so whether it is for IDAT or some other chunk. + */ +#define PNG_ZLIB_UNINITIALIZED 0 +#define PNG_ZLIB_FOR_IDAT 1 +#define PNG_ZLIB_FOR_TEXT 2 /* anything other than IDAT */ +#define PNG_ZLIB_USE_MASK 3 /* bottom two bits */ +#define PNG_ZLIB_IN_USE 4 /* a flag value */ + + png_uint_32 zlib_state; /* State of zlib initialization */ +/* End of material added at libpng 1.5.4 */ + int zlib_level; /* holds zlib compression level */ int zlib_method; /* holds zlib compression method */ int zlib_window_bits; /* holds zlib compression window bits */ int zlib_mem_level; /* holds zlib compression memory level */ int zlib_strategy; /* holds zlib compression strategy */ +#endif +/* Added at libpng 1.5.4 */ +#if defined(PNG_WRITE_COMPRESSED_TEXT_SUPPORTED) || \ + defined(PNG_WRITE_CUSTOMIZE_ZTXT_COMPRESSION_SUPPORTED) + int zlib_text_level; /* holds zlib compression level */ + int zlib_text_method; /* holds zlib compression method */ + int zlib_text_window_bits; /* holds zlib compression window bits */ + int zlib_text_mem_level; /* holds zlib compression memory level */ + int zlib_text_strategy; /* holds zlib compression strategy */ +#endif +/* End of material added at libpng 1.5.4 */ png_uint_32 width; /* width of image in pixels */ png_uint_32 height; /* height of image in pixels */ @@ -84,6 +111,7 @@ struct png_struct_def png_bytep avg_row; /* buffer to save "avg" row when filtering */ png_bytep paeth_row; /* buffer to save "Paeth" row when filtering */ png_row_info row_info; /* used for transformation routines */ + png_size_t info_rowbytes; /* Added in 1.5.4: cache of updated row bytes */ png_uint_32 idat_size; /* current IDAT size for read */ png_uint_32 crc; /* current chunk CRC value */ @@ -108,7 +136,8 @@ struct png_struct_def png_uint_16 filler; /* filler bytes for pixel expansion */ #endif -#ifdef PNG_bKGD_SUPPORTED +#if defined(PNG_bKGD_SUPPORTED) || defined(PNG_READ_BACKGROUND_SUPPORTED) ||\ + defined(PNG_READ_ALPHA_MODE_SUPPORTED) png_byte background_gamma_type; png_fixed_point background_gamma; png_color_16 background; /* background color in screen gamma space */ @@ -209,7 +238,7 @@ struct png_struct_def #endif #ifdef PNG_TIME_RFC1123_SUPPORTED - png_charp time_buffer; /* String to hold RFC 1123 time text */ + char time_buffer[29]; /* String to hold RFC 1123 time text */ #endif /* New members added in libpng-1.0.6 */ diff --git a/harbour/src/3rd/png/pngtrans.c b/harbour/src/3rd/png/pngtrans.c index cf7622cdc6..6a6908dcd2 100644 --- a/harbour/src/3rd/png/pngtrans.c +++ b/harbour/src/3rd/png/pngtrans.c @@ -1,7 +1,7 @@ /* pngtrans.c - transforms the data in a row (used by both readers and writers) * - * Last changed in libpng 1.5.2 [March 31, 2011] + * Last changed in libpng 1.5.4 [July 7, 2011] * Copyright (c) 1998-2011 Glenn Randers-Pehrson * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) @@ -442,7 +442,11 @@ png_do_strip_channel(png_row_infop row_info, png_bytep row, int at_start) /* At the start sp will point to the first byte to copy and dp to where * it is copied to. ep always points just beyond the end of the row, so * the loop simply copies (channels-1) channels until sp reaches ep. + * + * at_start: 0 -- convert AG, XG, ARGB, XRGB, AAGG, XXGG, etc. + * nonzero -- convert GA, GX, RGBA, RGBX, GGAA, RRGGBBXX, etc. */ + /* GA, GX, XG cases */ if (row_info->channels == 2) { @@ -450,7 +454,7 @@ png_do_strip_channel(png_row_infop row_info, png_bytep row, int at_start) { if (at_start) /* Skip initial filler */ ++sp; - else /* Skip initial channels and, for sp, the filler */ + else /* Skip initial channel and, for sp, the filler */ sp += 2, ++dp; /* For a 1 pixel wide image there is nothing to do */ @@ -462,9 +466,9 @@ png_do_strip_channel(png_row_infop row_info, png_bytep row, int at_start) else if (row_info->bit_depth == 16) { - if (at_start) + if (at_start) /* Skip initial filler */ sp += 2; - else + else /* Skip initial channel and, for sp, the filler */ sp += 4, dp += 2; while (sp < ep) @@ -502,9 +506,9 @@ png_do_strip_channel(png_row_infop row_info, png_bytep row, int at_start) else if (row_info->bit_depth == 16) { - if (at_start) + if (at_start) /* Skip initial filler */ sp += 2; - else + else /* Skip initial channels and, for sp, the filler */ sp += 8, dp += 6; while (sp < ep) diff --git a/harbour/src/3rd/png/pngwrite.c b/harbour/src/3rd/png/pngwrite.c index 9277a95162..d1af9e899f 100644 --- a/harbour/src/3rd/png/pngwrite.c +++ b/harbour/src/3rd/png/pngwrite.c @@ -1,7 +1,7 @@ /* pngwrite.c - general routines to write a PNG file * - * Last changed in libpng 1.5.1 [February 3, 2011] + * Last changed in libpng 1.5.4 [July 7, 2011] * Copyright (c) 1998-2011 Glenn Randers-Pehrson * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) @@ -99,8 +99,10 @@ png_write_info_before_PLTE(png_structp png_ptr, png_infop info_ptr) int keep = png_handle_as_unknown(png_ptr, up->name); if (keep != PNG_HANDLE_CHUNK_NEVER && - up->location && !(up->location & PNG_HAVE_PLTE) && + up->location && + !(up->location & PNG_HAVE_PLTE) && !(up->location & PNG_HAVE_IDAT) && + !(up->location & PNG_AFTER_IDAT) && ((up->name[3] & 0x20) || keep == PNG_HANDLE_CHUNK_ALWAYS || (png_ptr->flags & PNG_FLAG_KEEP_UNSAFE_CHUNKS))) { @@ -273,8 +275,10 @@ png_write_info(png_structp png_ptr, png_infop info_ptr) { int keep = png_handle_as_unknown(png_ptr, up->name); if (keep != PNG_HANDLE_CHUNK_NEVER && - up->location && (up->location & PNG_HAVE_PLTE) && + up->location && + (up->location & PNG_HAVE_PLTE) && !(up->location & PNG_HAVE_IDAT) && + !(up->location & PNG_AFTER_IDAT) && ((up->name[3] & 0x20) || keep == PNG_HANDLE_CHUNK_ALWAYS || (png_ptr->flags & PNG_FLAG_KEEP_UNSAFE_CHUNKS))) { @@ -380,7 +384,8 @@ png_write_end(png_structp png_ptr, png_infop info_ptr) { int keep = png_handle_as_unknown(png_ptr, up->name); if (keep != PNG_HANDLE_CHUNK_NEVER && - up->location && (up->location & PNG_AFTER_IDAT) && + up->location && + (up->location & PNG_AFTER_IDAT) && ((up->name[3] & 0x20) || keep == PNG_HANDLE_CHUNK_ALWAYS || (png_ptr->flags & PNG_FLAG_KEEP_UNSAFE_CHUNKS))) { @@ -462,10 +467,9 @@ png_create_write_struct_2,(png_const_charp user_png_ver, png_voidp error_ptr, png_structp png_ptr; #ifdef PNG_SETJMP_SUPPORTED #ifdef USE_FAR_KEYWORD - jmp_buf png_jmpbuf; + jmp_buf tmp_jmpbuf; #endif #endif - int i; png_debug(1, "in png_create_write_struct"); @@ -489,12 +493,12 @@ png_create_write_struct_2,(png_const_charp user_png_ver, png_voidp error_ptr, encounter a png_error() will longjmp here. Since the jmpbuf is then meaningless we abort instead of returning. */ #ifdef USE_FAR_KEYWORD - if (setjmp(png_jmpbuf)) + if (setjmp(tmp_jmpbuf)) #else if (setjmp(png_jmpbuf(png_ptr))) /* sets longjmp to match setjmp */ #endif #ifdef USE_FAR_KEYWORD - png_memcpy(png_jmpbuf(png_ptr), png_jmpbuf, png_sizeof(jmp_buf)); + png_memcpy(png_jmpbuf(png_ptr), tmp_jmpbuf, png_sizeof(jmp_buf)); #endif PNG_ABORT(); #endif @@ -504,49 +508,8 @@ png_create_write_struct_2,(png_const_charp user_png_ver, png_voidp error_ptr, #endif /* PNG_USER_MEM_SUPPORTED */ png_set_error_fn(png_ptr, error_ptr, error_fn, warn_fn); - if (user_png_ver) - { - i = 0; - do - { - if (user_png_ver[i] != png_libpng_ver[i]) - png_ptr->flags |= PNG_FLAG_LIBRARY_MISMATCH; - } while (png_libpng_ver[i++]); - } - - if (png_ptr->flags & PNG_FLAG_LIBRARY_MISMATCH) - { - /* Libpng 0.90 and later are binary incompatible with libpng 0.89, so - * we must recompile any applications that use any older library version. - * For versions after libpng 1.0, we will be compatible, so we need - * only check the first digit. - */ - if (user_png_ver == NULL || user_png_ver[0] != png_libpng_ver[0] || - (user_png_ver[0] == '1' && user_png_ver[2] != png_libpng_ver[2]) || - (user_png_ver[0] == '0' && user_png_ver[2] < '9')) - { -#ifdef PNG_CONSOLE_IO_SUPPORTED - char msg[80]; - - if (user_png_ver) - { - png_snprintf2(msg, 80, - "Application built with libpng-%.20s" - " but running with %.20s", - user_png_ver, - png_libpng_ver); - png_warning(png_ptr, msg); - } -#else - png_warning(png_ptr, - "Incompatible libpng version in application and library"); -#endif -#ifdef PNG_ERROR_NUMBERS_SUPPORTED - png_ptr->flags = 0; -#endif - png_cleanup_needed = 1; - } - } + if (!png_user_version_check(png_ptr, user_png_ver)) + png_cleanup_needed = 1; /* Initialize zbuf - compression buffer */ png_ptr->zbuf_size = PNG_ZBUF_SIZE; @@ -805,9 +768,11 @@ png_write_row(png_structp png_ptr, png_const_bytep row) } #endif +#ifdef PNG_WRITE_TRANSFORMS_SUPPORTED /* Handle other transformations */ if (png_ptr->transformations) png_do_write_transformations(png_ptr); +#endif #ifdef PNG_MNG_FEATURES_SUPPORTED /* Write filter_method 64 (intrapixel differencing) only if @@ -884,8 +849,6 @@ png_write_flush(png_structp png_ptr) { /* Write the IDAT and reset the zlib output buffer */ png_write_IDAT(png_ptr, png_ptr->zbuf, png_ptr->zbuf_size); - png_ptr->zstream.next_out = png_ptr->zbuf; - png_ptr->zstream.avail_out = (uInt)png_ptr->zbuf_size; wrote_IDAT = 1; } } while (wrote_IDAT == 1); @@ -896,8 +859,6 @@ png_write_flush(png_structp png_ptr) /* Write the IDAT and reset the zlib output buffer */ png_write_IDAT(png_ptr, png_ptr->zbuf, png_ptr->zbuf_size - png_ptr->zstream.avail_out); - png_ptr->zstream.next_out = png_ptr->zbuf; - png_ptr->zstream.avail_out = (uInt)png_ptr->zbuf_size; } png_ptr->flush_rows = 0; png_flush(png_ptr); @@ -983,7 +944,9 @@ png_write_destroy(png_structp png_ptr) jmp_buf tmp_jmp; /* Save jump buffer */ #endif png_error_ptr error_fn; +#ifdef PNG_WARNINGS_SUPPORTED png_error_ptr warning_fn; +#endif png_voidp error_ptr; #ifdef PNG_USER_MEM_SUPPORTED png_free_ptr free_fn; @@ -992,7 +955,8 @@ png_write_destroy(png_structp png_ptr) png_debug(1, "in png_write_destroy"); /* Free any memory zlib uses */ - deflateEnd(&png_ptr->zstream); + if (png_ptr->zlib_state != PNG_ZLIB_UNINITIALIZED) + deflateEnd(&png_ptr->zstream); /* Free our memory. png_free checks NULL for us. */ png_free(png_ptr, png_ptr->zbuf); @@ -1005,10 +969,6 @@ png_write_destroy(png_structp png_ptr) png_free(png_ptr, png_ptr->paeth_row); #endif -#ifdef PNG_TIME_RFC1123_SUPPORTED - png_free(png_ptr, png_ptr->time_buffer); -#endif - #ifdef PNG_WRITE_WEIGHTED_FILTER_SUPPORTED /* Use this to save a little code space, it doesn't free the filter_costs */ png_reset_filter_heuristics(png_ptr); @@ -1018,11 +978,13 @@ png_write_destroy(png_structp png_ptr) #ifdef PNG_SETJMP_SUPPORTED /* Reset structure */ - png_memcpy(tmp_jmp, png_ptr->png_jmpbuf, png_sizeof(jmp_buf)); + png_memcpy(tmp_jmp, png_ptr->longjmp_buffer, png_sizeof(jmp_buf)); #endif error_fn = png_ptr->error_fn; +#ifdef PNG_WARNINGS_SUPPORTED warning_fn = png_ptr->warning_fn; +#endif error_ptr = png_ptr->error_ptr; #ifdef PNG_USER_MEM_SUPPORTED free_fn = png_ptr->free_fn; @@ -1031,14 +993,16 @@ png_write_destroy(png_structp png_ptr) png_memset(png_ptr, 0, png_sizeof(png_struct)); png_ptr->error_fn = error_fn; +#ifdef PNG_WARNINGS_SUPPORTED png_ptr->warning_fn = warning_fn; +#endif png_ptr->error_ptr = error_ptr; #ifdef PNG_USER_MEM_SUPPORTED png_ptr->free_fn = free_fn; #endif #ifdef PNG_SETJMP_SUPPORTED - png_memcpy(png_ptr->png_jmpbuf, tmp_jmp, png_sizeof(jmp_buf)); + png_memcpy(png_ptr->longjmp_buffer, tmp_jmp, png_sizeof(jmp_buf)); #endif } @@ -1451,6 +1415,9 @@ png_set_compression_strategy(png_structp png_ptr, int strategy) png_ptr->zlib_strategy = strategy; } +/* If PNG_WRITE_OPTIMIZE_CMF_SUPPORTED is defined, libpng will use a + * smaller value of window_bits if it can do so safely. + */ void PNGAPI png_set_compression_window_bits(png_structp png_ptr, int window_bits) { @@ -1491,6 +1458,89 @@ png_set_compression_method(png_structp png_ptr, int method) png_ptr->zlib_method = method; } +/* The following were added to libpng-1.5.4 */ +#ifdef PNG_WRITE_CUSTOMIZE_ZTXT_COMPRESSION_SUPPORTED +void PNGAPI +png_set_text_compression_level(png_structp png_ptr, int level) +{ + png_debug(1, "in png_set_text_compression_level"); + + if (png_ptr == NULL) + return; + + png_ptr->flags |= PNG_FLAG_ZTXT_CUSTOM_LEVEL; + png_ptr->zlib_text_level = level; +} + +void PNGAPI +png_set_text_compression_mem_level(png_structp png_ptr, int mem_level) +{ + png_debug(1, "in png_set_text_compression_mem_level"); + + if (png_ptr == NULL) + return; + + png_ptr->flags |= PNG_FLAG_ZTXT_CUSTOM_MEM_LEVEL; + png_ptr->zlib_text_mem_level = mem_level; +} + +void PNGAPI +png_set_text_compression_strategy(png_structp png_ptr, int strategy) +{ + png_debug(1, "in png_set_text_compression_strategy"); + + if (png_ptr == NULL) + return; + + png_ptr->flags |= PNG_FLAG_ZTXT_CUSTOM_STRATEGY; + png_ptr->zlib_text_strategy = strategy; +} + +/* If PNG_WRITE_OPTIMIZE_CMF_SUPPORTED is defined, libpng will use a + * smaller value of window_bits if it can do so safely. + */ +void PNGAPI +png_set_text_compression_window_bits(png_structp png_ptr, int window_bits) +{ + if (png_ptr == NULL) + return; + + if (window_bits > 15) + png_warning(png_ptr, "Only compression windows <= 32k supported by PNG"); + + else if (window_bits < 8) + png_warning(png_ptr, "Only compression windows >= 256 supported by PNG"); + +#ifndef WBITS_8_OK + /* Avoid libpng bug with 256-byte windows */ + if (window_bits == 8) + { + png_warning(png_ptr, "Text compression window is being reset to 512"); + window_bits = 9; + } + +#endif + png_ptr->flags |= PNG_FLAG_ZTXT_CUSTOM_WINDOW_BITS; + png_ptr->zlib_text_window_bits = window_bits; +} + +void PNGAPI +png_set_text_compression_method(png_structp png_ptr, int method) +{ + png_debug(1, "in png_set_text_compression_method"); + + if (png_ptr == NULL) + return; + + if (method != 8) + png_warning(png_ptr, "Only compression method 8 is supported by PNG"); + + png_ptr->flags |= PNG_FLAG_ZTXT_CUSTOM_METHOD; + png_ptr->zlib_text_method = method; +} +#endif /* PNG_WRITE_CUSTOMIZE_ZTXT_COMPRESSION_SUPPORTED */ +/* end of API added to libpng-1.5.4 */ + void PNGAPI png_set_write_status_fn(png_structp png_ptr, png_write_status_ptr write_row_fn) { @@ -1557,7 +1607,7 @@ png_write_png(png_structp png_ptr, png_infop info_ptr, #endif #ifdef PNG_WRITE_FILLER_SUPPORTED - /* Pack XRGB/RGBX/ARGB/RGBA into * RGB (4 channels -> 3 channels) */ + /* Pack XRGB/RGBX/ARGB/RGBA into RGB (4 channels -> 3 channels) */ if (transforms & PNG_TRANSFORM_STRIP_FILLER_AFTER) png_set_filler(png_ptr, 0, PNG_FILLER_AFTER); diff --git a/harbour/src/3rd/png/pngwtran.c b/harbour/src/3rd/png/pngwtran.c index bbf4a25b92..124d708c4a 100644 --- a/harbour/src/3rd/png/pngwtran.c +++ b/harbour/src/3rd/png/pngwtran.c @@ -1,7 +1,7 @@ /* pngwtran.c - transforms the data in a row for PNG writers * - * Last changed in libpng 1.5.2 [March 31, 2011] + * Last changed in libpng 1.5.4 [July 7, 2011] * Copyright (c) 1998-2011 Glenn Randers-Pehrson * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) @@ -15,6 +15,7 @@ #ifdef PNG_WRITE_SUPPORTED +#ifdef PNG_WRITE_TRANSFORMS_SUPPORTED /* Transform the data according to the user's wishes. The order of * transformations is significant. */ @@ -45,7 +46,7 @@ png_do_write_transformations(png_structp png_ptr) #ifdef PNG_WRITE_FILLER_SUPPORTED if (png_ptr->transformations & PNG_FILLER) png_do_strip_channel(&(png_ptr->row_info), png_ptr->row_buf + 1, - !(png_ptr->flags & PNG_FILLER_AFTER)); + !(png_ptr->flags & PNG_FLAG_FILLER_AFTER)); #endif #ifdef PNG_WRITE_PACKSWAP_SUPPORTED @@ -563,6 +564,7 @@ png_do_write_invert_alpha(png_row_infop row_info, png_bytep row) } } #endif +#endif /* PNG_WRITE_TRANSFORMS_SUPPORTED */ #ifdef PNG_MNG_FEATURES_SUPPORTED /* Undoes intrapixel differencing */ diff --git a/harbour/src/3rd/png/pngwutil.c b/harbour/src/3rd/png/pngwutil.c index 2c35785aa1..3ae5ed4e2f 100644 --- a/harbour/src/3rd/png/pngwutil.c +++ b/harbour/src/3rd/png/pngwutil.c @@ -1,7 +1,7 @@ /* pngwutil.c - utilities to write a PNG file * - * Last changed in libpng 1.5.0 [January 6, 2011] + * Last changed in libpng 1.5.4 [July 7, 2011] * Copyright (c) 1998-2011 Glenn Randers-Pehrson * (Version 0.96 Copyright (c) 1996, 1997 Andreas Dilger) * (Version 0.88 Copyright (c) 1995, 1996 Guy Eric Schalnat, Group 42, Inc.) @@ -192,7 +192,149 @@ png_write_chunk_end(png_structp png_ptr) png_write_data(png_ptr, buf, (png_size_t)4); } -#if defined(PNG_WRITE_TEXT_SUPPORTED) || defined(PNG_WRITE_iCCP_SUPPORTED) +/* Initialize the compressor for the appropriate type of compression. */ +static void +png_zlib_claim(png_structp png_ptr, png_uint_32 state) +{ + if (!(png_ptr->zlib_state & PNG_ZLIB_IN_USE)) + { + /* If already initialized for 'state' do not re-init. */ + if (png_ptr->zlib_state != state) + { + int ret = Z_OK; + png_const_charp who = "-"; + + /* If actually initialized for another state do a deflateEnd. */ + if (png_ptr->zlib_state != PNG_ZLIB_UNINITIALIZED) + { + ret = deflateEnd(&png_ptr->zstream); + who = "end"; + png_ptr->zlib_state = PNG_ZLIB_UNINITIALIZED; + } + + /* zlib itself detects an incomplete state on deflateEnd */ + if (ret == Z_OK) switch (state) + { +# ifdef PNG_WRITE_COMPRESSED_TEXT_SUPPORTED + case PNG_ZLIB_FOR_TEXT: + ret = deflateInit2(&png_ptr->zstream, + png_ptr->zlib_text_level, png_ptr->zlib_text_method, + png_ptr->zlib_text_window_bits, + png_ptr->zlib_text_mem_level, png_ptr->zlib_text_strategy); + who = "text"; + break; +# endif + + case PNG_ZLIB_FOR_IDAT: + ret = deflateInit2(&png_ptr->zstream, png_ptr->zlib_level, + png_ptr->zlib_method, png_ptr->zlib_window_bits, + png_ptr->zlib_mem_level, png_ptr->zlib_strategy); + who = "IDAT"; + break; + + default: + png_error(png_ptr, "invalid zlib state"); + } + + if (ret == Z_OK) + png_ptr->zlib_state = state; + + else /* an error in deflateEnd or deflateInit2 */ + { + size_t pos = 0; + char msg[64]; + + pos = png_safecat(msg, sizeof msg, pos, + "zlib failed to initialize compressor ("); + pos = png_safecat(msg, sizeof msg, pos, who); + + switch (ret) + { + case Z_VERSION_ERROR: + pos = png_safecat(msg, sizeof msg, pos, ") version error"); + break; + + case Z_STREAM_ERROR: + pos = png_safecat(msg, sizeof msg, pos, ") stream error"); + break; + + case Z_MEM_ERROR: + pos = png_safecat(msg, sizeof msg, pos, ") memory error"); + break; + + default: + pos = png_safecat(msg, sizeof msg, pos, ") unknown error"); + break; + } + + png_error(png_ptr, msg); + } + } + + /* Here on success, claim the zstream: */ + png_ptr->zlib_state |= PNG_ZLIB_IN_USE; + } + + else + png_error(png_ptr, "zstream already in use (internal error)"); +} + +/* The opposite: release the stream. It is also reset, this API will warn on + * error but will not fail. + */ +static void +png_zlib_release(png_structp png_ptr) +{ + if (png_ptr->zlib_state & PNG_ZLIB_IN_USE) + { + int ret = deflateReset(&png_ptr->zstream); + + png_ptr->zlib_state &= ~PNG_ZLIB_IN_USE; + + if (ret != Z_OK) + { + png_const_charp err; + PNG_WARNING_PARAMETERS(p) + + switch (ret) + { + case Z_VERSION_ERROR: + err = "version"; + break; + + case Z_STREAM_ERROR: + err = "stream"; + break; + + case Z_MEM_ERROR: + err = "memory"; + break; + + default: + err = "unknown"; + break; + } + + png_warning_parameter_signed(p, 1, PNG_NUMBER_FORMAT_d, ret); + png_warning_parameter(p, 2, err); + + if (png_ptr->zstream.msg) + err = png_ptr->zstream.msg; + else + err = "[no zlib message]"; + + png_warning_parameter(p, 3, err); + + png_formatted_warning(png_ptr, p, + "zlib failed to reset compressor: @1(@2): @3"); + } + } + + else + png_warning(png_ptr, "zstream not in use (internal error)"); +} + +#ifdef PNG_WRITE_COMPRESSED_TEXT_SUPPORTED /* This pair of functions encapsulates the operation of (a) compressing a * text string, and (b) issuing it later as a series of chunk data writes. * The compression_state structure is shared context for these functions @@ -220,25 +362,22 @@ png_text_compress(png_structp png_ptr, comp->max_output_ptr = 0; comp->output_ptr = NULL; comp->input = NULL; - comp->input_len = 0; + comp->input_len = text_len; /* We may just want to pass the text right through */ if (compression == PNG_TEXT_COMPRESSION_NONE) { comp->input = (png_const_bytep)text; - comp->input_len = text_len; return((int)text_len); } if (compression >= PNG_TEXT_COMPRESSION_LAST) { -#ifdef PNG_CONSOLE_IO_SUPPORTED - char msg[50]; - png_snprintf(msg, 50, "Unknown compression type %d", compression); - png_warning(png_ptr, msg); -#else - png_warning(png_ptr, "Unknown compression type"); -#endif + PNG_WARNING_PARAMETERS(p) + + png_warning_parameter_signed(p, 1, PNG_NUMBER_FORMAT_d, + compression); + png_formatted_warning(png_ptr, p, "Unknown compression type @1"); } /* We can't write the chunk until we find out how much data we have, @@ -255,10 +394,12 @@ png_text_compress(png_structp png_ptr, * data, or if the input string is incredibly large (although this * wouldn't cause a failure, just a slowdown due to swapping). */ + png_zlib_claim(png_ptr, PNG_ZLIB_FOR_TEXT); /* Set up the compression buffers */ /* TODO: the following cast hides a potential overflow problem. */ png_ptr->zstream.avail_in = (uInt)text_len; + /* NOTE: assume zlib doesn't overwrite the input */ png_ptr->zstream.next_in = (Bytef *)text; png_ptr->zstream.avail_out = png_ptr->zbuf_size; @@ -418,6 +559,68 @@ png_write_compressed_data_out(png_structp png_ptr, compression_state *comp) return; } +#ifdef PNG_WRITE_OPTIMIZE_CMF_SUPPORTED + if (comp->input_len >= 2 && comp->input_len < 16384) + { + unsigned int z_cmf; /* zlib compression method and flags */ + + /* Optimize the CMF field in the zlib stream. This hack of the zlib + * stream is compliant to the stream specification. + */ + + if (comp->num_output_ptr) + z_cmf = comp->output_ptr[0][0]; + else + z_cmf = png_ptr->zbuf[0]; + + if ((z_cmf & 0x0f) == 8 && (z_cmf & 0xf0) <= 0x70) + { + unsigned int z_cinfo; + unsigned int half_z_window_size; + png_size_t uncompressed_text_size = comp->input_len; + + z_cinfo = z_cmf >> 4; + half_z_window_size = 1 << (z_cinfo + 7); + + while (uncompressed_text_size <= half_z_window_size && + half_z_window_size >= 256) + { + z_cinfo--; + half_z_window_size >>= 1; + } + + z_cmf = (z_cmf & 0x0f) | (z_cinfo << 4); + + if (comp->num_output_ptr) + { + + if (comp->output_ptr[0][0] != z_cmf) + { + int tmp; + + comp->output_ptr[0][0] = (png_byte)z_cmf; + tmp = comp->output_ptr[0][1] & 0xe0; + tmp += 0x1f - ((z_cmf << 8) + tmp) % 0x1f; + comp->output_ptr[0][1] = (png_byte)tmp; + } + } + else + { + int tmp; + + png_ptr->zbuf[0] = (png_byte)z_cmf; + tmp = png_ptr->zbuf[1] & 0xe0; + tmp += 0x1f - ((z_cmf << 8) + tmp) % 0x1f; + png_ptr->zbuf[1] = (png_byte)tmp; + } + } + + else + png_error(png_ptr, + "Invalid zlib compression method or flags in non-IDAT chunk"); + } +#endif /* PNG_WRITE_OPTIMIZE_CMF_SUPPORTED */ + /* Write saved output buffers, if any */ for (i = 0; i < comp->num_output_ptr; i++) { @@ -436,10 +639,9 @@ png_write_compressed_data_out(png_structp png_ptr, compression_state *comp) (png_size_t)(png_ptr->zbuf_size - png_ptr->zstream.avail_out)); /* Reset zlib for another zTXt/iTXt or image data */ - deflateReset(&png_ptr->zstream); - png_ptr->zstream.data_type = Z_BINARY; + png_zlib_release(png_ptr); } -#endif +#endif /* PNG_WRITE_COMPRESSED_TEXT_SUPPORTED */ /* Write the IHDR chunk, and update the png_struct with the necessary * information. Note that the rest of this code depends upon this @@ -451,7 +653,6 @@ png_write_IHDR(png_structp png_ptr, png_uint_32 width, png_uint_32 height, int interlace_type) { PNG_IHDR; - int ret; png_byte buf[13]; /* Buffer to store the IHDR info */ @@ -632,35 +833,35 @@ png_write_IHDR(png_structp png_ptr, png_uint_32 width, png_uint_32 height, if (!(png_ptr->flags & PNG_FLAG_ZLIB_CUSTOM_METHOD)) png_ptr->zlib_method = 8; - ret = deflateInit2(&png_ptr->zstream, png_ptr->zlib_level, - png_ptr->zlib_method, png_ptr->zlib_window_bits, - png_ptr->zlib_mem_level, png_ptr->zlib_strategy); +#ifdef PNG_WRITE_COMPRESSED_TEXT_SUPPORTED +#ifdef PNG_WRITE_CUSTOMIZE_ZTXT_COMPRESSION_SUPPORTED + if (!(png_ptr->flags & PNG_FLAG_ZTXT_CUSTOM_STRATEGY)) + png_ptr->zlib_text_strategy = Z_DEFAULT_STRATEGY; - if (ret != Z_OK) - { - if (ret == Z_VERSION_ERROR) - png_error(png_ptr, - "zlib failed to initialize compressor -- version error"); + if (!(png_ptr->flags & PNG_FLAG_ZTXT_CUSTOM_LEVEL)) + png_ptr->zlib_text_level = png_ptr->zlib_level; - if (ret == Z_STREAM_ERROR) - png_error(png_ptr, - "zlib failed to initialize compressor -- stream error"); + if (!(png_ptr->flags & PNG_FLAG_ZTXT_CUSTOM_MEM_LEVEL)) + png_ptr->zlib_text_mem_level = png_ptr->zlib_mem_level; - if (ret == Z_MEM_ERROR) - png_error(png_ptr, - "zlib failed to initialize compressor -- mem error"); + if (!(png_ptr->flags & PNG_FLAG_ZTXT_CUSTOM_WINDOW_BITS)) + png_ptr->zlib_text_window_bits = png_ptr->zlib_window_bits; - png_error(png_ptr, "zlib failed to initialize compressor"); - } + if (!(png_ptr->flags & PNG_FLAG_ZTXT_CUSTOM_METHOD)) + png_ptr->zlib_text_method = png_ptr->zlib_method; +#else + png_ptr->zlib_text_strategy = Z_DEFAULT_STRATEGY; + png_ptr->zlib_text_level = png_ptr->zlib_level; + png_ptr->zlib_text_mem_level = png_ptr->zlib_mem_level; + png_ptr->zlib_text_window_bits = png_ptr->zlib_window_bits; + png_ptr->zlib_text_method = png_ptr->zlib_method; +#endif /* PNG_WRITE_CUSTOMIZE_ZTXT_COMPRESSION_SUPPORTED */ +#endif /* PNG_WRITE_COMPRESSED_TEXT_SUPPORTED */ - png_ptr->zstream.next_out = png_ptr->zbuf; - png_ptr->zstream.avail_out = (uInt)png_ptr->zbuf_size; - /* libpng is not interested in zstream.data_type, so set it - * to a predefined value, to avoid its evaluation inside zlib - */ - png_ptr->zstream.data_type = Z_BINARY; + /* Record that the compressor has not yet been initialized. */ + png_ptr->zlib_state = PNG_ZLIB_UNINITIALIZED; - png_ptr->mode = PNG_HAVE_IHDR; + png_ptr->mode = PNG_HAVE_IHDR; /* not READY_FOR_ZTXT */ } /* Write the palette. We are careful not to trust png_color to be in the @@ -745,12 +946,15 @@ png_write_IDAT(png_structp png_ptr, png_bytep data, png_size_t length) png_debug(1, "in png_write_IDAT"); - /* Optimize the CMF field in the zlib stream. */ - /* This hack of the zlib stream is compliant to the stream specification. */ +#ifdef PNG_WRITE_OPTIMIZE_CMF_SUPPORTED if (!(png_ptr->mode & PNG_HAVE_IDAT) && png_ptr->compression_type == PNG_COMPRESSION_TYPE_BASE) { + /* Optimize the CMF field in the zlib stream. This hack of the zlib + * stream is compliant to the stream specification. + */ unsigned int z_cmf = data[0]; /* zlib compression method and flags */ + if ((z_cmf & 0x0f) == 8 && (z_cmf & 0xf0) <= 0x70) { /* Avoid memory underflows and multiplication overflows. @@ -761,11 +965,29 @@ png_write_IDAT(png_structp png_ptr, png_bytep data, png_size_t length) if (length >= 2 && png_ptr->height < 16384 && png_ptr->width < 16384) { + /* Compute the maximum possible length of the datastream */ + + /* Number of pixels, plus for each row a filter byte + * and possibly a padding byte, so increase the maximum + * size to account for these. + */ + unsigned int z_cinfo; + unsigned int half_z_window_size; png_uint_32 uncompressed_idat_size = png_ptr->height * ((png_ptr->width * png_ptr->channels * png_ptr->bit_depth + 15) >> 3); - unsigned int z_cinfo = z_cmf >> 4; - unsigned int half_z_window_size = 1 << (z_cinfo + 7); + + /* If it's interlaced, each block of 8 rows is sent as up to + * 14 rows, i.e., 6 additional rows, each with a filter byte + * and possibly a padding byte + */ + if (png_ptr->interlaced) + uncompressed_idat_size += ((png_ptr->height + 7)/8) * + (png_ptr->bit_depth < 8 ? 12 : 6); + + z_cinfo = z_cmf >> 4; + half_z_window_size = 1 << (z_cinfo + 7); + while (uncompressed_idat_size <= half_z_window_size && half_z_window_size >= 256) { @@ -790,9 +1012,17 @@ png_write_IDAT(png_structp png_ptr, png_bytep data, png_size_t length) png_error(png_ptr, "Invalid zlib compression method or flags in IDAT"); } +#endif /* PNG_WRITE_OPTIMIZE_CMF_SUPPORTED */ png_write_chunk(png_ptr, png_IDAT, data, length); png_ptr->mode |= PNG_HAVE_IDAT; + + /* Prior to 1.5.4 this code was replicated in every caller (except at the + * end, where it isn't technically necessary). Since this function has + * flushed the data we can safely reset the zlib output buffer here. + */ + png_ptr->zstream.next_out = png_ptr->zbuf; + png_ptr->zstream.avail_out = (uInt)png_ptr->zbuf_size; } /* Write an IEND chunk */ @@ -918,7 +1148,10 @@ png_write_iCCP(png_structp png_ptr, png_const_charp name, int compression_type, (png_size_t)(name_len + 2)); if (profile_len) + { + comp.input_len = profile_len; png_write_compressed_data_out(png_ptr, &comp); + } png_write_chunk_end(png_ptr); png_free(png_ptr, new_name); @@ -1313,15 +1546,11 @@ png_check_keyword(png_structp png_ptr, png_const_charp key, png_charpp new_key) if ((png_byte)*ikp < 0x20 || ((png_byte)*ikp > 0x7E && (png_byte)*ikp < 0xA1)) { -#ifdef PNG_CONSOLE_IO_SUPPORTED - char msg[40]; + PNG_WARNING_PARAMETERS(p) - png_snprintf(msg, 40, - "invalid keyword character 0x%02X", (png_byte)*ikp); - png_warning(png_ptr, msg); -#else - png_warning(png_ptr, "invalid character in keyword"); -#endif + png_warning_parameter_unsigned(p, 1, PNG_NUMBER_FORMAT_02x, + (png_byte)*ikp); + png_formatted_warning(png_ptr, p, "invalid keyword character 0x@1"); *dp = ' '; } @@ -1499,6 +1728,7 @@ png_write_zTXt(png_structp png_ptr, png_const_charp key, png_const_charp text, png_write_chunk_data(png_ptr, &buf, (png_size_t)1); /* Write the compressed data */ + comp.input_len = text_len; png_write_compressed_data_out(png_ptr, &comp); /* Close the chunk */ @@ -1871,6 +2101,7 @@ png_write_start_row(png_structp png_ptr) png_ptr->usr_width = png_ptr->width; } + png_zlib_claim(png_ptr, PNG_ZLIB_FOR_IDAT); png_ptr->zstream.avail_out = (uInt)png_ptr->zbuf_size; png_ptr->zstream.next_out = png_ptr->zbuf; } @@ -1992,7 +2223,7 @@ png_write_finish_row(png_structp png_ptr) png_ptr->zstream.avail_out); } - deflateReset(&png_ptr->zstream); + png_zlib_release(png_ptr); png_ptr->zstream.data_type = Z_BINARY; } @@ -2181,6 +2412,8 @@ png_do_write_interlace(png_row_infop row_info, png_bytep row, int pass) * been specified by the application, and then writes the row out with the * chosen filter. */ +static void png_write_filtered_row(png_structp png_ptr, png_bytep filtered_row); + #define PNG_MAXSUM (((png_uint_32)(-1)) >> 1) #define PNG_HISHIFT 10 #define PNG_LOMASK ((png_uint_32)0xffffL) @@ -2856,7 +3089,7 @@ png_write_find_filter(png_structp png_ptr, png_row_infop row_info) /* Do the actual writing of a previously filtered row. */ -void /* PRIVATE */ +static void png_write_filtered_row(png_structp png_ptr, png_bytep filtered_row) { png_size_t avail; @@ -2916,8 +3149,6 @@ png_write_filtered_row(png_structp png_ptr, png_bytep filtered_row) { /* Write the IDAT and reset the zlib output buffer */ png_write_IDAT(png_ptr, png_ptr->zbuf, png_ptr->zbuf_size); - png_ptr->zstream.next_out = png_ptr->zbuf; - png_ptr->zstream.avail_out = (uInt)png_ptr->zbuf_size; } /* Repeat until all data has been compressed */ } while (avail > 0 || png_ptr->zstream.avail_in > 0);