{"owner":"mystical.unicat","repo":"monorepo","channel":"main","channels":["main"],"can_delete":true,"inode":{"File":{"path":[{"basename":"engine","pos":"Q7P6NO6DTAU3Y.B4OAE","meta":576},{"basename":"ext","pos":"Q7P6NO6DTAU3Y.K4OQE","meta":576},{"basename":"sqlite","pos":"Q7P6NO6DTAU3Y.NAOQE","meta":576},{"basename":"ext","pos":"Q7P6NO6DTAU3Y.OW3MIBI","meta":576},{"basename":"fts5","pos":"Q7P6NO6DTAU3Y.WCAA4BQ","meta":576},{"basename":"fts5_test_tok.c","pos":"Q7P6NO6DTAU3Y.C5UDOBQ","meta":0}],"file":"/*\n** 2013 Apr 22\n**\n** The author disclaims copyright to this source code. In place of\n** a legal notice, here is a blessing:\n**\n** May you do good and not evil.\n** May you find forgiveness for yourself and forgive others.\n** May you share freely, never taking more than you give.\n**\n******************************************************************************\n**\n** This file contains code for the \"fts5tokenize\" virtual table module.\n** An fts5tokenize virtual table is created as follows:\n**\n** CREATE VIRTUAL TABLE USING fts5tokenize(\n** , , ...\n** );\n**\n** The table created has the following schema:\n**\n** CREATE TABLE (input HIDDEN, token, start, end, position)\n**\n** When queried, the query must include a WHERE clause of type:\n**\n** input = \n**\n** The virtual table module tokenizes this , using the FTS3 \n** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE \n** statement and returns one row for each token in the result. With\n** fields set as follows:\n**\n** input: Always set to a copy of \n** token: A token from the input.\n** start: Byte offset of the token within the input .\n** end: Byte offset of the byte immediately following the end of the\n** token within the input string.\n** pos: Token offset of token within input.\n**\n*/\n#if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5)\n\n#include \"fts5.h\"\n#include \n#include \n\ntypedef struct Fts5tokTable Fts5tokTable;\ntypedef struct Fts5tokCursor Fts5tokCursor;\ntypedef struct Fts5tokRow Fts5tokRow;\n\n/*\n** Virtual table structure.\n*/\nstruct Fts5tokTable {\n sqlite3_vtab base; /* Base class used by SQLite core */\n fts5_tokenizer tok; /* Tokenizer functions */\n Fts5Tokenizer *pTok; /* Tokenizer instance */\n};\n\n/*\n** A container for a rows values.\n*/\nstruct Fts5tokRow {\n char *zToken;\n int iStart;\n int iEnd;\n int iPos;\n};\n\n/*\n** Virtual table cursor structure.\n*/\nstruct Fts5tokCursor {\n sqlite3_vtab_cursor base; /* Base class used by SQLite core */\n int iRowid; /* Current 'rowid' value */\n char *zInput; /* Input string */\n int nRow; /* Number of entries in aRow[] */\n Fts5tokRow *aRow; /* Array of rows to return */\n};\n\nstatic void fts5tokDequote(char *z){\n char q = z[0];\n\n if( q=='[' || q=='\\'' || q=='\"' || q=='`' ){\n int iIn = 1;\n int iOut = 0;\n if( q=='[' ) q = ']'; \n\n while( z[iIn] ){\n if( z[iIn]==q ){\n if( z[iIn+1]!=q ){\n /* Character iIn was the close quote. */\n iIn++;\n break;\n }else{\n /* Character iIn and iIn+1 form an escaped quote character. Skip\n ** the input cursor past both and copy a single quote character \n ** to the output buffer. */\n iIn += 2;\n z[iOut++] = q;\n }\n }else{\n z[iOut++] = z[iIn++];\n }\n }\n\n z[iOut] = '\\0';\n }\n}\n\n/*\n** The second argument, argv[], is an array of pointers to nul-terminated\n** strings. This function makes a copy of the array and strings into a \n** single block of memory. It then dequotes any of the strings that appear\n** to be quoted.\n**\n** If successful, output parameter *pazDequote is set to point at the\n** array of dequoted strings and SQLITE_OK is returned. The caller is\n** responsible for eventually calling sqlite3_free() to free the array\n** in this case. Or, if an error occurs, an SQLite error code is returned.\n** The final value of *pazDequote is undefined in this case.\n*/\nstatic int fts5tokDequoteArray(\n int argc, /* Number of elements in argv[] */\n const char * const *argv, /* Input array */\n char ***pazDequote /* Output array */\n){\n int rc = SQLITE_OK; /* Return code */\n if( argc==0 ){\n *pazDequote = 0;\n }else{\n int i;\n int nByte = 0;\n char **azDequote;\n\n for(i=0; i0 ){\n zModule = azDequote[0];\n }\n\n rc = pApi->xFindTokenizer(pApi, zModule, &pTokCtx, &pTab->tok);\n if( rc==SQLITE_OK ){\n const char **azArg = (nDequote>1 ? (const char **)&azDequote[1] : 0);\n int nArg = nDequote>0 ? nDequote-1 : 0;\n rc = pTab->tok.xCreate(pTokCtx, azArg, nArg, &pTab->pTok);\n }\n }\n\n if( rc!=SQLITE_OK ){\n sqlite3_free(pTab);\n pTab = 0;\n }\n\n *ppVtab = (sqlite3_vtab*)pTab;\n sqlite3_free(azDequote);\n return rc;\n}\n\n/*\n** This function does the work for both the xDisconnect and xDestroy methods.\n** These tables have no persistent representation of their own, so xDisconnect\n** and xDestroy are identical operations.\n*/\nstatic int fts5tokDisconnectMethod(sqlite3_vtab *pVtab){\n Fts5tokTable *pTab = (Fts5tokTable *)pVtab;\n if( pTab->pTok ){\n pTab->tok.xDelete(pTab->pTok);\n }\n sqlite3_free(pTab);\n return SQLITE_OK;\n}\n\n/*\n** xBestIndex - Analyze a WHERE and ORDER BY clause.\n*/\nstatic int fts5tokBestIndexMethod(\n sqlite3_vtab *pVTab, \n sqlite3_index_info *pInfo\n){\n int i;\n\n for(i=0; inConstraint; i++){\n if( pInfo->aConstraint[i].usable \n && pInfo->aConstraint[i].iColumn==0 \n && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ \n ){\n pInfo->idxNum = 1;\n pInfo->aConstraintUsage[i].argvIndex = 1;\n pInfo->aConstraintUsage[i].omit = 1;\n pInfo->estimatedCost = 1;\n return SQLITE_OK;\n }\n }\n\n pInfo->idxNum = 0;\n assert( pInfo->estimatedCost>1000000.0 );\n\n return SQLITE_OK;\n}\n\n/*\n** xOpen - Open a cursor.\n*/\nstatic int fts5tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){\n Fts5tokCursor *pCsr;\n\n pCsr = (Fts5tokCursor *)sqlite3_malloc(sizeof(Fts5tokCursor));\n if( pCsr==0 ){\n return SQLITE_NOMEM;\n }\n memset(pCsr, 0, sizeof(Fts5tokCursor));\n\n *ppCsr = (sqlite3_vtab_cursor *)pCsr;\n return SQLITE_OK;\n}\n\n/*\n** Reset the tokenizer cursor passed as the only argument. As if it had\n** just been returned by fts5tokOpenMethod().\n*/\nstatic void fts5tokResetCursor(Fts5tokCursor *pCsr){\n int i;\n for(i=0; inRow; i++){\n sqlite3_free(pCsr->aRow[i].zToken);\n }\n sqlite3_free(pCsr->zInput);\n sqlite3_free(pCsr->aRow);\n pCsr->zInput = 0;\n pCsr->aRow = 0;\n pCsr->nRow = 0;\n pCsr->iRowid = 0;\n}\n\n/*\n** xClose - Close a cursor.\n*/\nstatic int fts5tokCloseMethod(sqlite3_vtab_cursor *pCursor){\n Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;\n fts5tokResetCursor(pCsr);\n sqlite3_free(pCsr);\n return SQLITE_OK;\n}\n\n/*\n** xNext - Advance the cursor to the next row, if any.\n*/\nstatic int fts5tokNextMethod(sqlite3_vtab_cursor *pCursor){\n Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;\n pCsr->iRowid++;\n return SQLITE_OK;\n}\n\nstatic int fts5tokCb(\n void *pCtx, /* Pointer to Fts5tokCursor */\n int tflags, /* Mask of FTS5_TOKEN_* flags */\n const char *pToken, /* Pointer to buffer containing token */\n int nToken, /* Size of token in bytes */\n int iStart, /* Byte offset of token within input text */\n int iEnd /* Byte offset of end of token within input text */\n){\n Fts5tokCursor *pCsr = (Fts5tokCursor*)pCtx;\n Fts5tokRow *pRow;\n\n if( (pCsr->nRow & (pCsr->nRow-1))==0 ){\n int nNew = pCsr->nRow ? pCsr->nRow*2 : 32;\n Fts5tokRow *aNew;\n aNew = (Fts5tokRow*)sqlite3_realloc64(pCsr->aRow, nNew*sizeof(Fts5tokRow));\n if( aNew==0 ) return SQLITE_NOMEM;\n memset(&aNew[pCsr->nRow], 0, sizeof(Fts5tokRow)*(nNew-pCsr->nRow));\n pCsr->aRow = aNew;\n }\n\n pRow = &pCsr->aRow[pCsr->nRow];\n pRow->iStart = iStart;\n pRow->iEnd = iEnd;\n if( pCsr->nRow ){\n pRow->iPos = pRow[-1].iPos + ((tflags & FTS5_TOKEN_COLOCATED) ? 0 : 1);\n }\n pRow->zToken = sqlite3_malloc(nToken+1);\n if( pRow->zToken==0 ) return SQLITE_NOMEM;\n memcpy(pRow->zToken, pToken, nToken);\n pRow->zToken[nToken] = 0;\n pCsr->nRow++;\n\n return SQLITE_OK;\n}\n\n/*\n** xFilter - Initialize a cursor to point at the start of its data.\n*/\nstatic int fts5tokFilterMethod(\n sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */\n int idxNum, /* Strategy index */\n const char *idxStr, /* Unused */\n int nVal, /* Number of elements in apVal */\n sqlite3_value **apVal /* Arguments for the indexing scheme */\n){\n int rc = SQLITE_ERROR;\n Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;\n Fts5tokTable *pTab = (Fts5tokTable *)(pCursor->pVtab);\n\n fts5tokResetCursor(pCsr);\n if( idxNum==1 ){\n const char *zByte = (const char *)sqlite3_value_text(apVal[0]);\n int nByte = sqlite3_value_bytes(apVal[0]);\n pCsr->zInput = sqlite3_malloc(nByte+1);\n if( pCsr->zInput==0 ){\n rc = SQLITE_NOMEM;\n }else{\n if( nByte>0 ) memcpy(pCsr->zInput, zByte, nByte);\n pCsr->zInput[nByte] = 0;\n rc = pTab->tok.xTokenize(\n pTab->pTok, (void*)pCsr, 0, zByte, nByte, fts5tokCb\n );\n }\n }\n\n if( rc!=SQLITE_OK ) return rc;\n return fts5tokNextMethod(pCursor);\n}\n\n/*\n** xEof - Return true if the cursor is at EOF, or false otherwise.\n*/\nstatic int fts5tokEofMethod(sqlite3_vtab_cursor *pCursor){\n Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;\n return (pCsr->iRowid>pCsr->nRow);\n}\n\n/*\n** xColumn - Return a column value.\n*/\nstatic int fts5tokColumnMethod(\n sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */\n sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */\n int iCol /* Index of column to read value from */\n){\n Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;\n Fts5tokRow *pRow = &pCsr->aRow[pCsr->iRowid-1];\n\n /* CREATE TABLE x(input, token, start, end, position) */\n switch( iCol ){\n case 0:\n sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT);\n break;\n case 1:\n sqlite3_result_text(pCtx, pRow->zToken, -1, SQLITE_TRANSIENT);\n break;\n case 2:\n sqlite3_result_int(pCtx, pRow->iStart);\n break;\n case 3:\n sqlite3_result_int(pCtx, pRow->iEnd);\n break;\n default:\n assert( iCol==4 );\n sqlite3_result_int(pCtx, pRow->iPos);\n break;\n }\n return SQLITE_OK;\n}\n\n/*\n** xRowid - Return the current rowid for the cursor.\n*/\nstatic int fts5tokRowidMethod(\n sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */\n sqlite_int64 *pRowid /* OUT: Rowid value */\n){\n Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;\n *pRowid = (sqlite3_int64)pCsr->iRowid;\n return SQLITE_OK;\n}\n\n/*\n** Register the fts5tok module with database connection db. Return SQLITE_OK\n** if successful or an error code if sqlite3_create_module() fails.\n*/\nint sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){\n static const sqlite3_module fts5tok_module = {\n 0, /* iVersion */\n fts5tokConnectMethod, /* xCreate */\n fts5tokConnectMethod, /* xConnect */\n fts5tokBestIndexMethod, /* xBestIndex */\n fts5tokDisconnectMethod, /* xDisconnect */\n fts5tokDisconnectMethod, /* xDestroy */\n fts5tokOpenMethod, /* xOpen */\n fts5tokCloseMethod, /* xClose */\n fts5tokFilterMethod, /* xFilter */\n fts5tokNextMethod, /* xNext */\n fts5tokEofMethod, /* xEof */\n fts5tokColumnMethod, /* xColumn */\n fts5tokRowidMethod, /* xRowid */\n 0, /* xUpdate */\n 0, /* xBegin */\n 0, /* xSync */\n 0, /* xCommit */\n 0, /* xRollback */\n 0, /* xFindFunction */\n 0, /* xRename */\n 0, /* xSavepoint */\n 0, /* xRelease */\n 0, /* xRollbackTo */\n 0 /* xShadowName */\n };\n int rc; /* Return code */\n\n rc = sqlite3_create_module(db, \"fts5tokenize\", &fts5tok_module, (void*)pApi);\n return rc;\n}\n\n#endif /* defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5) */\n"}},"token":"uBAPNlFlXCphNs6T9R1j8mHmQYPCXLEJ+XgphGZnmxA=","login":null}