mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-06-30 16:28:30 +00:00
Upgrade SQLite to 3.40 (#699)
This commit is contained in:
Paul Kulchenko
GitHub
parent
bcae817215
commit
0dc0758574
151 changed files with 27917 additions and 22169 deletions
130
third_party/sqlite3/fts3.c
vendored
130
third_party/sqlite3/fts3.c
vendored
|
|
@ -61,7 +61,7 @@
|
|||
** A doclist (document list) holds a docid-sorted list of hits for a
|
||||
** given term. Doclists hold docids and associated token positions.
|
||||
** A docid is the unique integer identifier for a single document.
|
||||
** A position is the index of a word within the document. The first
|
||||
** A position is the index of a word within the document. The first
|
||||
** word of the document has a position of 0.
|
||||
**
|
||||
** FTS3 used to optionally store character offsets using a compile-time
|
||||
|
|
@ -86,7 +86,7 @@
|
|||
**
|
||||
** Here, array { X } means zero or more occurrences of X, adjacent in
|
||||
** memory. A "position" is an index of a token in the token stream
|
||||
** generated by the tokenizer. Note that POS_END and POS_COLUMN occur
|
||||
** generated by the tokenizer. Note that POS_END and POS_COLUMN occur
|
||||
** in the same logical place as the position element, and act as sentinals
|
||||
** ending a position list array. POS_END is 0. POS_COLUMN is 1.
|
||||
** The positions numbers are not stored literally but rather as two more
|
||||
|
|
@ -110,7 +110,7 @@
|
|||
** a document record consists of a docid followed by a position-list and
|
||||
** a doclist consists of one or more document records.
|
||||
**
|
||||
** A bare doclist omits the position information, becoming an
|
||||
** A bare doclist omits the position information, becoming an
|
||||
** array of varint-encoded docids.
|
||||
**
|
||||
**** Segment leaf nodes ****
|
||||
|
|
@ -287,9 +287,8 @@
|
|||
** query logic likewise merges doclists so that newer data knocks out
|
||||
** older data.
|
||||
*/
|
||||
/* clang-format off */
|
||||
|
||||
#include "third_party/sqlite3/fts3Int.inc"
|
||||
#include "third_party/sqlite3/fts3Int.h"
|
||||
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
|
||||
|
||||
#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE)
|
||||
|
|
@ -300,31 +299,33 @@
|
|||
#include "libc/mem/mem.h"
|
||||
#include "libc/stdio/stdio.h"
|
||||
#include "libc/str/str.h"
|
||||
#include "third_party/sqlite3/fts3.inc"
|
||||
#ifndef SQLITE_CORE
|
||||
#include "third_party/sqlite3/sqlite3ext.h"
|
||||
SQLITE_EXTENSION_INIT1
|
||||
|
||||
#include "third_party/sqlite3/fts3.h"
|
||||
#ifndef SQLITE_CORE
|
||||
# include "third_party/sqlite3/sqlite3ext.h"
|
||||
SQLITE_EXTENSION_INIT1
|
||||
#endif
|
||||
|
||||
typedef struct Fts3HashWrapper Fts3HashWrapper;
|
||||
struct Fts3HashWrapper {
|
||||
Fts3Hash hash; /* Hash table */
|
||||
int nRef; /* Number of pointers to this object */
|
||||
};
|
||||
|
||||
static int fts3EvalNext(Fts3Cursor *pCsr);
|
||||
static int fts3EvalStart(Fts3Cursor *pCsr);
|
||||
static int fts3TermSegReaderCursor(
|
||||
Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **);
|
||||
|
||||
#ifndef SQLITE_AMALGAMATION
|
||||
# if defined(SQLITE_DEBUG)
|
||||
int sqlite3Fts3Always(int b) { assert( b ); return b; }
|
||||
int sqlite3Fts3Never(int b) { assert( !b ); return b; }
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/*
|
||||
** This variable is set to false when running tests for which the on disk
|
||||
** structures should not be corrupt. Otherwise, true. If it is false, extra
|
||||
** assert() conditions in the fts3 code are activated - conditions that are
|
||||
** only true if it is guaranteed that the fts3 database is not corrupt.
|
||||
*/
|
||||
#ifdef SQLITE_DEBUG
|
||||
int sqlite3_fts3_may_be_corrupt = 1;
|
||||
#endif
|
||||
|
||||
/*
|
||||
** Write a 64-bit variable-length integer to memory starting at p[0].
|
||||
|
|
@ -1175,7 +1176,7 @@ static int fts3InitVtab(
|
|||
sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
|
||||
char **pzErr /* Write any error message here */
|
||||
){
|
||||
Fts3Hash *pHash = (Fts3Hash *)pAux;
|
||||
Fts3Hash *pHash = &((Fts3HashWrapper*)pAux)->hash;
|
||||
Fts3Table *p = 0; /* Pointer to allocated vtab */
|
||||
int rc = SQLITE_OK; /* Return code */
|
||||
int i; /* Iterator variable */
|
||||
|
|
@ -1895,7 +1896,7 @@ static int fts3ScanInteriorNode(
|
|||
char *zBuffer = 0; /* Buffer to load terms into */
|
||||
i64 nAlloc = 0; /* Size of allocated buffer */
|
||||
int isFirstTerm = 1; /* True when processing first term on page */
|
||||
sqlite3_int64 iChild; /* Block id of child node to descend to */
|
||||
u64 iChild; /* Block id of child node to descend to */
|
||||
int nBuffer = 0; /* Total term size */
|
||||
|
||||
/* Skip over the 'height' varint that occurs at the start of every
|
||||
|
|
@ -1911,8 +1912,8 @@ static int fts3ScanInteriorNode(
|
|||
** table, then there are always 20 bytes of zeroed padding following the
|
||||
** nNode bytes of content (see sqlite3Fts3ReadBlock() for details).
|
||||
*/
|
||||
zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
|
||||
zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
|
||||
zCsr += sqlite3Fts3GetVarintU(zCsr, &iChild);
|
||||
zCsr += sqlite3Fts3GetVarintU(zCsr, &iChild);
|
||||
if( zCsr>zEnd ){
|
||||
return FTS_CORRUPT_VTAB;
|
||||
}
|
||||
|
|
@ -1965,20 +1966,20 @@ static int fts3ScanInteriorNode(
|
|||
*/
|
||||
cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
|
||||
if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){
|
||||
*piFirst = iChild;
|
||||
*piFirst = (i64)iChild;
|
||||
piFirst = 0;
|
||||
}
|
||||
|
||||
if( piLast && cmp<0 ){
|
||||
*piLast = iChild;
|
||||
*piLast = (i64)iChild;
|
||||
piLast = 0;
|
||||
}
|
||||
|
||||
iChild++;
|
||||
};
|
||||
|
||||
if( piFirst ) *piFirst = iChild;
|
||||
if( piLast ) *piLast = iChild;
|
||||
if( piFirst ) *piFirst = (i64)iChild;
|
||||
if( piLast ) *piLast = (i64)iChild;
|
||||
|
||||
finish_scan:
|
||||
sqlite3_free(zBuffer);
|
||||
|
|
@ -2885,7 +2886,7 @@ static int fts3TermSelectMerge(
|
|||
**
|
||||
** Similar padding is added in the fts3DoclistOrMerge() function.
|
||||
*/
|
||||
pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1);
|
||||
pTS->aaOutput[0] = sqlite3_malloc64((i64)nDoclist + FTS3_VARINT_MAX + 1);
|
||||
pTS->anOutput[0] = nDoclist;
|
||||
if( pTS->aaOutput[0] ){
|
||||
memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
|
||||
|
|
@ -3584,14 +3585,20 @@ static int fts3SetHasStat(Fts3Table *p){
|
|||
*/
|
||||
static int fts3BeginMethod(sqlite3_vtab *pVtab){
|
||||
Fts3Table *p = (Fts3Table*)pVtab;
|
||||
int rc;
|
||||
UNUSED_PARAMETER(pVtab);
|
||||
assert( p->pSegments==0 );
|
||||
assert( p->nPendingData==0 );
|
||||
assert( p->inTransaction!=1 );
|
||||
TESTONLY( p->inTransaction = 1 );
|
||||
TESTONLY( p->mxSavepoint = -1; );
|
||||
p->nLeafAdd = 0;
|
||||
return fts3SetHasStat(p);
|
||||
rc = fts3SetHasStat(p);
|
||||
#ifdef SQLITE_DEBUG
|
||||
if( rc==SQLITE_OK ){
|
||||
p->inTransaction = 1;
|
||||
p->mxSavepoint = -1;
|
||||
}
|
||||
#endif
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -4004,9 +4011,12 @@ static const sqlite3_module fts3Module = {
|
|||
** allocated for the tokenizer hash table.
|
||||
*/
|
||||
static void hashDestroy(void *p){
|
||||
Fts3Hash *pHash = (Fts3Hash *)p;
|
||||
sqlite3Fts3HashClear(pHash);
|
||||
sqlite3_free(pHash);
|
||||
Fts3HashWrapper *pHash = (Fts3HashWrapper *)p;
|
||||
pHash->nRef--;
|
||||
if( pHash->nRef<=0 ){
|
||||
sqlite3Fts3HashClear(&pHash->hash);
|
||||
sqlite3_free(pHash);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -4036,7 +4046,7 @@ void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
|
|||
*/
|
||||
int sqlite3Fts3Init(sqlite3 *db){
|
||||
int rc = SQLITE_OK;
|
||||
Fts3Hash *pHash = 0;
|
||||
Fts3HashWrapper *pHash = 0;
|
||||
const sqlite3_tokenizer_module *pSimple = 0;
|
||||
const sqlite3_tokenizer_module *pPorter = 0;
|
||||
#ifndef SQLITE_DISABLE_FTS3_UNICODE
|
||||
|
|
@ -4064,23 +4074,24 @@ int sqlite3Fts3Init(sqlite3 *db){
|
|||
sqlite3Fts3PorterTokenizerModule(&pPorter);
|
||||
|
||||
/* Allocate and initialize the hash-table used to store tokenizers. */
|
||||
pHash = sqlite3_malloc(sizeof(Fts3Hash));
|
||||
pHash = sqlite3_malloc(sizeof(Fts3HashWrapper));
|
||||
if( !pHash ){
|
||||
rc = SQLITE_NOMEM;
|
||||
}else{
|
||||
sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
|
||||
sqlite3Fts3HashInit(&pHash->hash, FTS3_HASH_STRING, 1);
|
||||
pHash->nRef = 0;
|
||||
}
|
||||
|
||||
/* Load the built-in tokenizers into the hash table */
|
||||
if( rc==SQLITE_OK ){
|
||||
if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
|
||||
|| sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
|
||||
if( sqlite3Fts3HashInsert(&pHash->hash, "simple", 7, (void *)pSimple)
|
||||
|| sqlite3Fts3HashInsert(&pHash->hash, "porter", 7, (void *)pPorter)
|
||||
|
||||
#ifndef SQLITE_DISABLE_FTS3_UNICODE
|
||||
|| sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode)
|
||||
|| sqlite3Fts3HashInsert(&pHash->hash, "unicode61", 10, (void *)pUnicode)
|
||||
#endif
|
||||
#ifdef SQLITE_ENABLE_ICU
|
||||
|| (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
|
||||
|| (pIcu && sqlite3Fts3HashInsert(&pHash->hash, "icu", 4, (void *)pIcu))
|
||||
#endif
|
||||
){
|
||||
rc = SQLITE_NOMEM;
|
||||
|
|
@ -4089,7 +4100,7 @@ int sqlite3Fts3Init(sqlite3 *db){
|
|||
|
||||
#ifdef SQLITE_TEST
|
||||
if( rc==SQLITE_OK ){
|
||||
rc = sqlite3Fts3ExprInitTestInterface(db, pHash);
|
||||
rc = sqlite3Fts3ExprInitTestInterface(db, &pHash->hash);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -4098,23 +4109,26 @@ int sqlite3Fts3Init(sqlite3 *db){
|
|||
** module with sqlite.
|
||||
*/
|
||||
if( SQLITE_OK==rc
|
||||
&& SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
|
||||
&& SQLITE_OK==(rc=sqlite3Fts3InitHashTable(db,&pHash->hash,"fts3_tokenizer"))
|
||||
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
|
||||
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
|
||||
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))
|
||||
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2))
|
||||
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
|
||||
){
|
||||
pHash->nRef++;
|
||||
rc = sqlite3_create_module_v2(
|
||||
db, "fts3", &fts3Module, (void *)pHash, hashDestroy
|
||||
);
|
||||
if( rc==SQLITE_OK ){
|
||||
pHash->nRef++;
|
||||
rc = sqlite3_create_module_v2(
|
||||
db, "fts4", &fts3Module, (void *)pHash, 0
|
||||
db, "fts4", &fts3Module, (void *)pHash, hashDestroy
|
||||
);
|
||||
}
|
||||
if( rc==SQLITE_OK ){
|
||||
rc = sqlite3Fts3InitTok(db, (void *)pHash);
|
||||
pHash->nRef++;
|
||||
rc = sqlite3Fts3InitTok(db, (void *)pHash, hashDestroy);
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
|
@ -4123,7 +4137,7 @@ int sqlite3Fts3Init(sqlite3 *db){
|
|||
/* An error has occurred. Delete the hash table and return the error code. */
|
||||
assert( rc!=SQLITE_OK );
|
||||
if( pHash ){
|
||||
sqlite3Fts3HashClear(pHash);
|
||||
sqlite3Fts3HashClear(&pHash->hash);
|
||||
sqlite3_free(pHash);
|
||||
}
|
||||
return rc;
|
||||
|
|
@ -4292,8 +4306,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
|
|||
char *aPoslist = 0; /* Position list for deferred tokens */
|
||||
int nPoslist = 0; /* Number of bytes in aPoslist */
|
||||
int iPrev = -1; /* Token number of previous deferred token */
|
||||
|
||||
assert( pPhrase->doclist.bFreeList==0 );
|
||||
char *aFree = (pPhrase->doclist.bFreeList ? pPhrase->doclist.pList : 0);
|
||||
|
||||
for(iToken=0; iToken<pPhrase->nToken; iToken++){
|
||||
Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
|
||||
|
|
@ -4307,6 +4320,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
|
|||
|
||||
if( pList==0 ){
|
||||
sqlite3_free(aPoslist);
|
||||
sqlite3_free(aFree);
|
||||
pPhrase->doclist.pList = 0;
|
||||
pPhrase->doclist.nList = 0;
|
||||
return SQLITE_OK;
|
||||
|
|
@ -4327,6 +4341,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
|
|||
nPoslist = (int)(aOut - aPoslist);
|
||||
if( nPoslist==0 ){
|
||||
sqlite3_free(aPoslist);
|
||||
sqlite3_free(aFree);
|
||||
pPhrase->doclist.pList = 0;
|
||||
pPhrase->doclist.nList = 0;
|
||||
return SQLITE_OK;
|
||||
|
|
@ -4359,13 +4374,14 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
|
|||
nDistance = iPrev - nMaxUndeferred;
|
||||
}
|
||||
|
||||
aOut = (char *)sqlite3_malloc(nPoslist+8);
|
||||
aOut = (char *)sqlite3Fts3MallocZero(nPoslist+FTS3_BUFFER_PADDING);
|
||||
if( !aOut ){
|
||||
sqlite3_free(aPoslist);
|
||||
return SQLITE_NOMEM;
|
||||
}
|
||||
|
||||
pPhrase->doclist.pList = aOut;
|
||||
assert( p1 && p2 );
|
||||
if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){
|
||||
pPhrase->doclist.bFreeList = 1;
|
||||
pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList);
|
||||
|
|
@ -4378,6 +4394,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
|
|||
}
|
||||
}
|
||||
|
||||
if( pPhrase->doclist.pList!=aFree ) sqlite3_free(aFree);
|
||||
return SQLITE_OK;
|
||||
}
|
||||
#endif /* SQLITE_DISABLE_FTS4_DEFERRED */
|
||||
|
|
@ -4470,7 +4487,7 @@ void sqlite3Fts3DoclistPrev(
|
|||
|
||||
assert( nDoclist>0 );
|
||||
assert( *pbEof==0 );
|
||||
assert( p || *piDocid==0 );
|
||||
assert_fts3_nc( p || *piDocid==0 );
|
||||
assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) );
|
||||
|
||||
if( p==0 ){
|
||||
|
|
@ -4726,7 +4743,7 @@ static int fts3EvalIncrPhraseNext(
|
|||
if( bEof==0 ){
|
||||
int nList = 0;
|
||||
int nByte = a[p->nToken-1].nList;
|
||||
char *aDoclist = sqlite3_malloc(nByte+FTS3_BUFFER_PADDING);
|
||||
char *aDoclist = sqlite3_malloc64((i64)nByte+FTS3_BUFFER_PADDING);
|
||||
if( !aDoclist ) return SQLITE_NOMEM;
|
||||
memcpy(aDoclist, a[p->nToken-1].pList, nByte+1);
|
||||
memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING);
|
||||
|
|
@ -5120,16 +5137,15 @@ static int fts3EvalStart(Fts3Cursor *pCsr){
|
|||
#ifndef SQLITE_DISABLE_FTS4_DEFERRED
|
||||
if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){
|
||||
Fts3TokenAndCost *aTC;
|
||||
Fts3Expr **apOr;
|
||||
aTC = (Fts3TokenAndCost *)sqlite3_malloc64(
|
||||
sizeof(Fts3TokenAndCost) * nToken
|
||||
+ sizeof(Fts3Expr *) * nOr * 2
|
||||
);
|
||||
apOr = (Fts3Expr **)&aTC[nToken];
|
||||
|
||||
if( !aTC ){
|
||||
rc = SQLITE_NOMEM;
|
||||
}else{
|
||||
Fts3Expr **apOr = (Fts3Expr **)&aTC[nToken];
|
||||
int ii;
|
||||
Fts3TokenAndCost *pTC = aTC;
|
||||
Fts3Expr **ppOr = apOr;
|
||||
|
|
@ -5335,8 +5351,8 @@ static void fts3EvalNextRow(
|
|||
Fts3Expr *pRight = pExpr->pRight;
|
||||
sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
|
||||
|
||||
assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
|
||||
assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
|
||||
assert_fts3_nc( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
|
||||
assert_fts3_nc( pRight->bStart || pLeft->iDocid==pRight->iDocid );
|
||||
|
||||
if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
|
||||
fts3EvalNextRow(pCsr, pLeft, pRc);
|
||||
|
|
@ -5553,11 +5569,10 @@ static int fts3EvalTestExpr(
|
|||
|
||||
default: {
|
||||
#ifndef SQLITE_DISABLE_FTS4_DEFERRED
|
||||
if( pCsr->pDeferred
|
||||
&& (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
|
||||
){
|
||||
if( pCsr->pDeferred && (pExpr->bDeferred || (
|
||||
pExpr->iDocid==pCsr->iPrevId && pExpr->pPhrase->doclist.pList
|
||||
))){
|
||||
Fts3Phrase *pPhrase = pExpr->pPhrase;
|
||||
assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
|
||||
if( pExpr->bDeferred ){
|
||||
fts3EvalInvalidatePoslist(pPhrase);
|
||||
}
|
||||
|
|
@ -5974,6 +5989,9 @@ int sqlite3Fts3EvalPhrasePoslist(
|
|||
if( bEofSave==0 && pNear->iDocid==iDocid ) break;
|
||||
}
|
||||
assert( rc!=SQLITE_OK || pPhrase->bIncr==0 );
|
||||
if( rc==SQLITE_OK && pNear->bEof!=bEofSave ){
|
||||
rc = FTS_CORRUPT_VTAB;
|
||||
}
|
||||
}
|
||||
if( bTreeEof ){
|
||||
while( rc==SQLITE_OK && !pNear->bEof ){
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue